#!/usr/bin/perl
use strict;
use warnings;
use Data::Dumper;
use Getopt::Long qw(:config no_ignore_case );
use MySQL::Sandbox;

my $DEBUG = $MySQL::Sandbox::DEBUG;

for my $prog (qw( make_sandbox 
                make_replication_sandbox 
                make_multiple_sandbox 
                make_multiple_custom_sandbox ) ) {
    unless ( exists_in_path ($prog) ) {
        die "script <$prog> not found\n";
    } 
}

#
# defaults
#
# my @versions = ( '5.0.51', '5.0.64', '5.1.23', '5.1.24', '5.1.25', '5.1.26', '6.0.6' );
my @versions = ( '5.0.67', '5.1.30');
my $verbose = $DEBUG || $ENV{'VERBOSE'} || 0;

my %tests = (
    single      => 1,
    replication => 1,
    circular    => 1,
    multiple    => 1,
    custom      => 1,
    tuning      => 1,
    smoke       => 0,
);

my %custom_tests = ();

my ($user_tests, $user_versions, $get_help);


GetOptions( 
   "tarball|versions=s" => \$user_versions,
   "tests=s"    => \$user_tests, 
   "verbose"    => \$verbose,
   "help|h"     => \$get_help,
) or help();

help() if $get_help;

if ($user_versions) {
    my @new_versions = grep {$_} split /,/, $user_versions;
    die "at least one version is required\n" unless @new_versions;
    @versions =();
    # ensuring that each version is used only once.
    @new_versions = 
        map     { $_->[0] } # sorting with the Schwartzian Transform
        sort    { $a->[1] cmp $b->[1] }
        map     { /(\d+)\.(\d+)\.(\d+)/; 
                    [ $_, sprintf('%02d-%02d-%02d',$1,$2,$3)] } 
        keys    %{{ map { $_, 1} @new_versions }};
    for my $ver (@new_versions) {
            push @versions, $ver;
    }
}
for my $ver (@versions) {
    $ver =~ s/^~/$ENV{HOME}/;
    unless (( -d "/opt/mysql/$ver") 
            or ( -d "$ENV{HOME}/opt/mysql/$ver") 
            or ( -f $ver )) {
        print "version $ver not found in either /opt/mysql or $ENV{HOME}/opt/mysql\n";
        die "use --versions to list the versions you want to test\n";
    }
}

if ($user_tests) {
    my @todo = grep {$_} split /,/, $user_tests;
    die "at least one test is required\n" unless @todo;
    my %new_tests;
    for my $t (@todo) {
        if (exists $tests{$t} ) {
            $new_tests{$t}++;
        }
        else {
            die "unrecognized test <$t>\n";
        }
    }
    for my $t (keys %tests) {
        if ( exists $new_tests{$t} ) {
            $tests{$t} = 1;
        }
        else {
            $tests{$t} = 0;
        }
    }
}

my $sandbox_home = "$ENV{'HOME'}/sandboxes";
my %test_results = (
    run     => 0,
    passed  => 0,
    failed  => 0,
    skipped => 0,
);
#
# cleaning up existing sandbox directory
#
if ( $ENV{'SANDBOX_HOME'} ) {
    $sandbox_home = $ENV{'SANDBOX_HOME'};
}
else {
    $ENV{'SANDBOX_HOME'} = $sandbox_home;
}
my $sh_stop_all = "$sandbox_home/stop_all";

if ( -x $sh_stop_all ) {
    system($sh_stop_all)
}

# 
# setting the current sandbox directory for this test
#
$sandbox_home = "$ENV{'HOME'}/test_sb";
$ENV{'SANDBOX_HOME'} = $sandbox_home;
$sh_stop_all = "$sandbox_home/stop_all";
my $sh_clear_all = "$sandbox_home/clear_all";
my $sh_start_all = "$sandbox_home/start_all";
my $sh_use_all = "$sandbox_home/use_all";

# 
# cleaning up the test directory if exists
#
if ( -x $sh_stop_all ) {
    system($sh_stop_all);
    system qq(rm -rf $sandbox_home) ;
}

#
# checking if there are other servers running
#
my $how_many_mysqld = get_number_of_processes('mysqld');
my $how_many_mysqld_safe = get_number_of_processes('mysqld_safe');
printf "** currently there are (%d) mysqld processes and (%d) mysqld_safe processes\n", 
    $how_many_mysqld, 
    $how_many_mysqld_safe;

#
# starting the tests
#
for my $ver (@versions) {
    my $bare_version = get_bare_version ($ver);
    my $version = $bare_version;
    $version =~ s/\./_/g;

    if ($tests{'single'}) {
        run_single_test($ver, $version, $bare_version);
    }

    if ($tests{'replication'} ) {
        run_replication_test($ver, $version, $bare_version);
    }

    if ($tests{'circular'}) {
        run_circular_test($ver, $version, $bare_version);
    }

    if ($tests{'multiple'}) {
        run_multiple_test($ver, $version, $bare_version);
    }
}

if ($tests{'custom'}) {
    run_custom_test();
}

my $summaries  = 0;
for my $test (keys %tests) {
    if ($tests{$test} && ($test ne 'tuning') && ($test ne 'smoke' )) {
        $summaries = 1;
        last;
    }
}

if ($summaries) {
    run_summary_tests();
}

if ($tests{tuning}) {
    run_tuning_test();
}

if ($tests{smoke}) {
    for my $ver ( @versions ) {
        if (-f "$sandbox_home/stop_all") {
            system qq($sandbox_home/stop_all); 
            system qq(rm -rf $sandbox_home/*) ;
        }
        run_smoke_test($ver);
    }
}


printf "*** Executed %d tests. Passed %d (%5.2f%%). Failed %d (%5.2f%%)\n",
        $test_results{run},
        $test_results{passed},
        $test_results{passed} / $test_results{run} * 100,
        $test_results{failed},
        $test_results{failed} / $test_results{run} * 100
        ;

# 
# ROUTINES
#
 
# 
# get_exec_result 
#
# runs a shell command and returns the output
#
sub get_exec_result {
    my ($cmd, $verbose) = @_; 
    print "(shell) $cmd\n" if $verbose;
    my $output = qx($cmd );
    if ($?) {
        die ("error executing $cmd ($!)\n");
    }
    return $output;
}

#
# get_sql_result
#
#  runs a SQL command and returns the output
#
sub get_sql_result {
    my ($sb, $query, $verbose) = @_;
    print "(sql) $query\n" if $verbose;
    if ( -f "$sb/use" ) {
        # print qq(<echo "$query" | $sb/use -N -B >\n);
        my $output = qx(echo "$query" | $sb/use -N -B );
        if (defined $output) {
            chomp $output;
        }
        else {
            $output = '';
        }
        if ($verbose && $verbose > 1) {
            print "$output\n";
        }
        die "error executing query $query on sandbox $sb\n" if $?;
        return $output;
    }
    else {
        die "can't find a 'use' command on $sb\n";
    }
}

 
#
# get_number_of_processes
#
# returns the number of processes for a given name
#
sub get_number_of_processes {
    my ($proc_name) = @_;

    my $grep_cmd = 'ps -ef | grep -w %s | grep -v "grep -w %s" | wc -l ';
    my $cmd = sprintf($grep_cmd, $proc_name, $proc_name );
    print "$cmd\n" if ($verbose && ($verbose > 1));
    my $how_many = get_exec_result($cmd, 0);
    return $how_many;
}

#
# help
#
# displays option for this program
#
sub help {
my $HELP = <<"HELP";

test for MySQL Sandbox
usage: $0 [options]
    --versions=version1[,version2,version3]
      uses specific versions for testing. 
      currently: (@{[join ",", @versions]})

    --tarball=/path/to/tarball
      it's an alias for --versions

    --tests=testname[,testname,testname]
      executes specific tests.
      currently: (@{[join ",", grep {$tests{$_}} keys %tests]})

    --verbose
      shows the commands executed during tests

    --help
      shows this help

HELP
    print $HELP;
    exit(1);
}

#
# TEST ROUTINES
#

#
# ok
#
# evaluates a condition and prints a ok/not ok message
#
sub ok ($;$) {
    my ($condition, $msg) = @_;
    # print Dumper \@_;
    $msg = '***' unless defined $msg;
    $test_results{run}++;
    if ($condition) {
        $test_results{passed}++;
    }
    else {
        print "not ";
        $test_results{failed}++;
    }
    print "ok - $msg\n";
    if ($verbose) { die "halting the test on verbose\n" unless $condition; }
    return $condition;
}

sub run_single_test {
    my ($ver, $version, $bare_version) = @_;
    my $single_sandbox = get_exec_result(
            "make_sandbox $ver --no_confirm", 
            $verbose);
    ok( $single_sandbox =~ /sandbox server started/,
            "single SB ($bare_version) started" );
    my $sql_result = get_sql_result(
            "$sandbox_home/msb_$version", 
            'select version(), @@server_id',
            $verbose );
    ok( $sql_result =~ /$bare_version/, 
            "single SB ($bare_version) SQL " );
    $sql_result = get_sql_result(
            "$sandbox_home/msb_$version", 
            'create database \`a-a\`', # Bug#278394 - this will fail on 'clear' if not fixed
            $verbose );
 
}

sub run_replication_test {
    my ($ver, $version, $bare_version) = @_;
    my $replication_sandbox = get_exec_result(
            "make_replication_sandbox $ver",
            $verbose);
    ok( $replication_sandbox !~ /not started yet/, 
            "replication sandbox ($bare_version) started");
    my $sql_result = get_sql_result(
            "$sandbox_home/rsandbox_$version/master", 
            'select version(), @@server_id', 
            $verbose );
    ok( ($sql_result =~ /$bare_version/), 
            "replication SB master ($bare_version) SQL - version" );
    ok( ($sql_result =~ /\b1\s*$/), 
            "replication SB master ($bare_version) SQL - server_id" );

    sleep 1;
    $sql_result = get_sql_result(
            "$sandbox_home/rsandbox_$version/node1", 
            'select version(), @@server_id', 
            $verbose );
    ok( ($sql_result =~ /$bare_version/), 
            "replication SB slave1 ($bare_version) SQL - version" );
    ok( ($sql_result =~ /\b101\s*$/), 
            "replication SB slave1 ($bare_version) SQL - server_id" );

    $sql_result = get_sql_result(
            "$sandbox_home/rsandbox_$version/node2", 
            'select version(), @@server_id',
            $verbose );
    ok( ($sql_result =~ /$bare_version/), 
            "replication SB slave2 ($bare_version) SQL - version" );
    ok(  ($sql_result =~ /\b102\s*$/), 
            "replication SB slave2 ($bare_version) SQL - server_id" );
    $sql_result = get_sql_result(
            "$sandbox_home/rsandbox_$version/master",
           q{drop table if exists test.t1; create table test.t1 (id int); show tables from test},
          $verbose); 
    ok( ($sql_result =~ /t1/) , 
            "replication SB - table created on master" );

    $sql_result = get_sql_result(
            "$sandbox_home/rsandbox_$version/node1",
           q{show tables from test},
           $verbose);
    ok( ($sql_result =~ /t1/), 
            "replication SB - table exists on slave1" );
    $sql_result = get_sql_result(
            "$sandbox_home/rsandbox_$version/node2",
           q{show tables from test},
           $verbose);
    ok( ($sql_result =~ /t1/), 
            "replication SB - table exists on slave2" );
}

sub run_circular_test {
    my ($ver, $version, $bare_version) = @_;
    my $replication_sandbox = get_exec_result(
            "make_replication_sandbox --how_many_slaves=3 --topology=circular $ver",
            $verbose);
    ok( $replication_sandbox !~ /not started yet/, 
            "circular replication sandbox ($bare_version) started");

    for my $node ( 1 .. 3) {
        #$verbose = 1;
        my $sql_result = get_sql_result(
            "$sandbox_home/rcsandbox_$version/node$node", 
            'show slave status\G', 
            $verbose );
        ok( $sql_result =~ /IO_Running.+Yes/ , 
            "circular replication SB node$node ($bare_version) SQL - IO Running" );
        ok( $sql_result =~ /SQL_Running.+Yes/ , 
            "circular replication SB node$node ($bare_version) SQL - SQL Running" );
        sleep 1;
    }
}

sub run_multiple_test {
    my ($ver, $version, $bare_version) = @_;
    my $multiple_sandbox = get_exec_result(
            "make_multiple_sandbox $ver",
            $verbose);
    ok( $multiple_sandbox !~ /not started yet/, 
            "multiple sandbox ($bare_version) started");

    for my $node (1 .. 3 ) {
        my $sql_result = get_sql_result(
                "$sandbox_home/multi_msb_$version/node$node", 
                'select version(), @@server_id',
                $verbose );
        ok($sql_result =~ /$bare_version/, 
                "multiple SB node $node ($bare_version) SQL - version" );
        ok( $sql_result =~ /\b10$node\s*$/, 
                "multiple SB node $node ($bare_version) SQL - server_id" );
    }
}

sub run_custom_test {
    my $custom_sandbox = get_exec_result(
            "make_multiple_custom_sandbox @versions",
            $verbose);
    ok($custom_sandbox !~ /not started yet/, 
                "custom sandbox ( @{[map {get_bare_version($_)} @versions]} ) started");
    my $custom_dir = '';
    if ($custom_sandbox =~ /group directory installed on (\S+)/) {
        $custom_dir = $1;
        ok(-d $custom_dir, 'custom group directory exists')
            or die "custom group directory not created\n";
    }
    else {
        die "can't find custom group directory\n";
    }
    my $counter = 0;
    for my $ver (@versions) {
        my $bare_version = get_bare_version($ver);
        $counter++;
        my $sql_result = get_sql_result(
                    "$custom_dir/node$counter", 
                    'select version(), @@server_id',
                    $verbose );
        ok($sql_result =~ /$bare_version/, 
                    "multiple custom SB node $counter ($bare_version) SQL - version" );
        ok($sql_result =~ /\b10$counter\s*$/, 
                    "multiple custom SB node $counter ($bare_version) SQL - server_id" );
    }
}

sub run_summary_tests {
    my $new_mysqld_procs = get_number_of_processes('mysqld');
    my $new_mysqld_safe_procs = get_number_of_processes('mysqld_safe');

    printf "** created (%d) mysqld processes and (%d) mysqld_safe processes\n", 
        $new_mysqld_procs - $how_many_mysqld, 
        $new_mysqld_safe_procs - $how_many_mysqld_safe;

    my $instances = 
               (  1 * $tests{single}  # single 
                + 3 * $tests{replication}  # replicated
                + 3 * $tests{circular}  # circular
                + 3 * $tests{multiple} # multiple
              );
    my $expected_processes = 
                $instances * scalar(@versions)
                + (scalar(@versions) * $tests{custom})  ; # custom counts only once

    ok( $expected_processes == ($new_mysqld_safe_procs - $how_many_mysqld_safe), 
            "expected processes ($expected_processes)" );

    print "** stopping all - please wait\n";
    my $stop_all = get_exec_result("$sandbox_home/stop_all", $verbose);

    $new_mysqld_procs = get_number_of_processes('mysqld');
    $new_mysqld_safe_procs = get_number_of_processes('mysqld_safe');

    printf "** (%d) mysqld processes and (%d) mysqld_safe processes\n", 
        $new_mysqld_procs - $how_many_mysqld, 
        $new_mysqld_safe_procs - $how_many_mysqld_safe;
    ok( ($new_mysqld_safe_procs - $how_many_mysqld_safe ) == 0,
            'expected processes (0)' );

    print "** starting all - please wait\n";
    my $start_all = get_exec_result("$sandbox_home/start_all", $verbose);

    $new_mysqld_procs = get_number_of_processes('mysqld');
    $new_mysqld_safe_procs = get_number_of_processes('mysqld_safe');
    printf "** created (%d) mysqld processes and (%d) mysqld_safe processes\n", 
        $new_mysqld_procs - $how_many_mysqld, 
        $new_mysqld_safe_procs - $how_many_mysqld_safe;

    ok( $expected_processes == ($new_mysqld_safe_procs - $how_many_mysqld_safe), 
            "expected processes ($expected_processes)" );

    unless ($ENV{PRESERVE_TESTS}) { 
        print "** cleaning up - please wait\n";
        my $clear_all = get_exec_result("$sandbox_home/clear_all", $verbose);

        $new_mysqld_procs = get_number_of_processes('mysqld');
        $new_mysqld_safe_procs = get_number_of_processes('mysqld_safe');

        printf "** (%d) mysqld processes and (%d) mysqld_safe processes\n", 
            $new_mysqld_procs - $how_many_mysqld, 
            $new_mysqld_safe_procs - $how_many_mysqld_safe;
        ok( ($new_mysqld_safe_procs - $how_many_mysqld_safe ) == 0,
            'expected processes (0)' ) 
            or die "can't continue without a clean environment\n";

        system qq(rm -rf $sandbox_home/*) ;
    }
}

sub run_tuning_test {
    my $ver;
    for my $v (@versions) {
        $ver = $v;
        if ($ver =~ /^[^34]/) {
            last;
        }
    }
    if ($ver =~ /^[34]/) {
        print "skipping tuning test. It requires version >=5\n";
        return;
    }
    my $bare_version = get_bare_version($ver);
    my $version = $bare_version;
    $version =~ s/\./_/g;

    my $single_sandbox = get_exec_result("make_sandbox $ver --no_confirm -c skip-innodb -c sql_mode=strict_all_tables");
    my $sql_result = get_sql_result(
           "$sandbox_home/msb_$version", 
            'show engines',
            $verbose );
    ok( ($sql_result !~ /innodb\s*yes/i), 
            "single SB with option skip-innodb ($bare_version) SQL " )
        or print "engines: $sql_result\n";
     
    $sql_result = get_sql_result(
           "$sandbox_home/msb_$version", 
            q(show variables like 'SQL_MODE'),
            $verbose );
    ok( ($sql_result =~ /STRICT_ALL_TABLES/i),
            "single SB with option sql_mode ($bare_version) SQL " )
        or print "SQL_MODE: $sql_result\n";

    $sql_result = get_sql_result(
           "$sandbox_home/msb_$version", 
            q(create database xyz; show databases like 'xyz'),
            $verbose);
    ok ( $sql_result =~ /xyz/,
            "single SB ($bare_version) - create database");
    my $clear_sandbox = get_exec_result("$sandbox_home/msb_$version/clear");
    ok ($clear_sandbox !~ /error/i, "single SB ($bare_version) - clear result" );

    # my $sandbox_dirs = get_exec_result("ls -d $sandbox_home/msb_$version/data/*/ | wc -l ");
    my $sandbox_dirs = how_many_dirs("$sandbox_home/msb_$version/data");

    ok($sandbox_dirs == 2 , "single SB ($bare_version) - effective clear ");

    print "** cleaning up - please wait\n";
    my $clear_all = get_exec_result("$sandbox_home/clear_all", $verbose);
}

sub get_bare_version {
    my ($ver) = @_;
    $ver =~ s{.*/}{};
    if ($ver =~ /(\d+\.\d+\.\d+)/) {
        return $1;
    }
    else {
        die "'$ver'  does not contain a valid version\n";
    }
}

sub run_smoke_test {
    my ($ver) = @_;
    if ($ver =~ /^[34]/) {
        print "skipping tuning test. It requires version >=5\n";
        return;
    }
    my $bare_version = get_bare_version($ver);
    my $version = $bare_version;
    $version =~ s/\./_/g;
    if ($ver =~ m{(.+)/[^/]+(?:tgz|tar\.gz)$}) {
        my $bindir = $1;
        if ( -d "$bindir/$bare_version" ) {
            system "rm -rf $bindir/$bare_version";
        } 
    }

    my $previous_mysqld = get_number_of_processes('mysqld');
    my $previous_mysqld_safe = get_number_of_processes('mysqld_safe');

    my $single_sandbox = get_exec_result("make_sandbox $ver --no_confirm");
    ok( $single_sandbox =~ /sandbox server started/,
            "single SB ($bare_version) started" );

    my $first_mysqld = get_number_of_processes('mysqld');
    my $first_mysqld_safe = get_number_of_processes('mysqld_safe');

    ok($first_mysqld_safe > $previous_mysqld_safe, 'mysqld_safe started') ;
    ok($first_mysqld > $previous_mysqld, 'mysqld started') ;

    my $sql_result = get_sql_result(
            "$sandbox_home/msb_$version", 
            q{show variables like 'pid_file'},
            $verbose);
    my $pid_file ;
    if ($sql_result =~ m{\s*(\S+\.pid)} ) {
        $pid_file = $1;
    }
    ok ($pid_file , 'pid_file found')
        or die "can't find pid file\n";

    my $pid = get_pid($pid_file)
        or die "can't get PID from $pid_file\n" ;
    my $pid_ts1 = get_pid_timestamp($pid_file)
        or die "can't get timestamp for file $pid_file\n";

    my $kill_result = get_exec_result("kill -9 $pid");
    ok (! $kill_result, "mysqld killed");

    my $timeout = 20;
    my $counter = 0;
    my $second_mysqld = 0;
    my $started ;
    while (! $started ) {
        $counter++;
        if ($counter >= $timeout) {
            last;
        }
        $second_mysqld = get_number_of_processes('mysqld');
        $started = $second_mysqld >= $first_mysqld;
        unless ($started) {
            print "-- waiting for mysqld to restart ($counter)\n";
            sleep 1;
        }
    }
    my $pid_ts2 = get_pid_timestamp($pid_file);
    $counter = 0;
    # print "PID_TS ==== $pid_ts1 $pid_ts2\n";
    while ((!$pid_ts2) or ($pid_ts2 eq $pid_ts1)) {
        $counter++;
        print ">> waiting for mysqld to restart ($counter)\n";
        sleep 1; 
        $pid_ts2 = get_pid($pid_file);
        if ($counter > $timeout) {
            die "error recovering killed process\n";
        }
    }
    ok ($second_mysqld >= $first_mysqld, "new mysqld process created (1)") ;
    my $secondpid = get_pid($pid_file)
        or die "can't get PID from $pid_file\n" ;

    chomp $secondpid;
    print "previous pid ($pid) current pid ($secondpid)\n" if $verbose;
    ok ($secondpid ne $pid, "new mysqld process created (2)");

    $sql_result = get_sql_result(
            "$sandbox_home/msb_$version", 
            'select version()',
            $verbose );

    ok( $sql_result =~ /$bare_version/, 
            "single SB ($bare_version) version (1)" );

    $sql_result = get_sql_result(
            "$sandbox_home/msb_$version", 
            q{show variables like 'version'},
            $verbose );
    ok( $sql_result =~ /$bare_version/, 
            "single SB ($bare_version) version (2) " );

    my $visual_version = $sql_result;
    chomp $visual_version;
    $sql_result = get_sql_result(
            "$sandbox_home/msb_$version", 
            'select @@version_comment limit 1',
            $verbose );
    my $visual_comment = $sql_result;
    chomp $visual_comment;

    $sql_result = get_sql_result(
           "$sandbox_home/msb_$version", 
            'HELP SELECT',
            $verbose );
    # print "HELP: $sql_result\n";
    ok( $sql_result && (! ( $sql_result =~ /nothing\s*found/i)), 
            "single SB ($bare_version) HELP tables filled " );
     
    $sql_result = get_sql_result(
           "$sandbox_home/msb_$version", 
            q(create database install_test; show databases like 'install_test'),
            $verbose);
    ok ( $sql_result =~ /install_test/,
            "single SB ($bare_version) - create database");
    
    my @test_engines = (
        {
            min_version => 5 ,
            engine  => 'innodb'
        },
        {
            min_version => 6 ,
            engine  => 'falcon'
        },
        {
            min_version => 6 ,
            engine  => 'maria'
        },
    );

    for my $test_engine (@test_engines) {
        my $major_version = substr($version,0,1);
        if ($major_version <  $test_engine->{min_version} ) {
            next;
        }
        my $engine = $test_engine->{engine};
    $sql_result = get_sql_result(
           "$sandbox_home/msb_$version", 
           qq{drop table if exists install_test.testib;
            create table install_test.testib ( 
            id int UNSIGNED NOT NULL AUTO_INCREMENT , 
            nr int UNSIGNED NOT NULL , 
            PRIMARY KEY ( id)
            ) engine=$engine; show tables from install_test },
            $verbose);
    ok ( $sql_result =~ /testib/,
            "single SB ($bare_version) - create table");

    $sql_result = get_sql_result(
           "$sandbox_home/msb_$version", 
           q{select engine from information_schema.tables
             where table_schema='install_test'
             and table_name = 'testib'
            },
            $verbose);
    my $right_engine=   ($sql_result and ($sql_result =~ /$engine/i));

    
    ok ( $right_engine,
            "single SB ($bare_version) - create table - check engine ($engine)")
        or print "result:<$sql_result>\n" ;

    $sql_result = get_sql_result(
           "$sandbox_home/msb_$version", 
           q{alter table install_test.testib add index nr (nr);
            insert into install_test.testib(id,nr) values ( '1','1');
            insert into install_test.testib(id,nr) values ( '2','2');
            insert into install_test.testib(id,nr) values ( '3','2');
            select * from install_test.testib where nr=2 order by id asc;},
            $verbose);
    ok ( $sql_result =~ /2\t2\s*3\t2/, 'smoke 1');

    }
    $sql_result = get_sql_result(
           "$sandbox_home/msb_$version", 
           q{select * from install_test.testib where nr=2 order by id desc;},
            $verbose);
    ok ( $sql_result =~ /3\t2\s*2\t2/, 'smoke 2');

    $sql_result = get_sql_result(
           "$sandbox_home/msb_$version", 
            q{select count(*) from install_test.testib},
            $verbose);
    ok ($sql_result =~ /^3$/, 'rows in table'); 

    $sql_result = get_sql_result(
           "$sandbox_home/msb_$version", 
            q{truncate table install_test.testib; },
            $verbose);
    $sql_result = get_sql_result(
           "$sandbox_home/msb_$version", 
            q{select count(*) from install_test.testib},
            $verbose);
    ok ($sql_result =~ /^0$/, 'table truncation'); 

    $sql_result = get_sql_result(
           "$sandbox_home/msb_$version", 
            q{ drop table install_test.testib; },
            $verbose);

    $sql_result = get_sql_result(
           "$sandbox_home/msb_$version", 
            q{select count(*) from information_schema.tables where table_schema='install_test'},
            $verbose);

    ok ($sql_result =~ /^0$/, 'tables in install_test'); 

    $sql_result = get_sql_result(
           "$sandbox_home/msb_$version", 
            q{ drop database install_test;},
            $verbose);
    # my $sandbox_dirs = get_exec_result("ls -d $sandbox_home/msb_$version/data/*/ | wc -l ", $verbose);
    my $sandbox_dirs = how_many_dirs("$sandbox_home/msb_$version/data");

    ok($sandbox_dirs == 2 , "single SB ($bare_version) - effective clean up ");

    print "** cleaning up - please wait\n";
    my $clear = get_exec_result("$sandbox_home/msb_$version/clear", $verbose);
    if ($ver =~ m{(.+)/[^/]+(?:tgz|tar\.gz)$}) {
        my $bindir = $1;
        if ( -d "$bindir/$bare_version" ) {
            system "rm -rf $bindir/$bare_version";
            system "rm -rf $sandbox_home/msb_$version";
        } 
    }

    print "\n[VISUAL IDENTIFICATION]\n";
    print "\tcheck version: <$visual_version>\n";
    print "\tcheck comment: <$visual_comment>\n\n";
}

sub get_pid {
    my ($pfile) = @_;
    my $timeout = 5;
    my $counter = 0;
    my $PFILE;
    while (! $PFILE) {
        eval {
            open( $PFILE, q{<}, $pfile)
                or die;
        };
        next if $@;
        last if $PFILE;
        if ($counter >= $timeout) {
            die "can't open $pfile\n";
        }
        else {
            sleep 1;
        }
        $counter++;
    }
    print Dumper $PFILE if $verbose;
    die "can't open $pfile\n" unless $PFILE;
    my $pid = <$PFILE>;
    close $PFILE;
    chomp $pid;
    if ($pid &&  ($pid =~ /^\d+$/) ) {
        return $pid;
    }
    return 0;
}

sub get_pid_timestamp {
    my ($pidfile) = @_;

    my @stats = stat $pidfile
        or return 0;
    #    or die "can't get timestamp for file $pidfile ($!)\n";    
    return $stats[8];
}

sub how_many_dirs {
    my ($path) = @_;
    my @subdirs = glob("$path/*/");
    my $dir_count =0;
    for (@subdirs) {
        $dir_count++ if -d $_;
    }
    return $dir_count;
}

sub exists_in_path {
    my ($fname) = @_;
    my @paths = split /:/, $ENV{PATH};
    for my $path (@paths) {
        $path =~ s{/$}{};
        if ( -f "$path/$fname") {
            return 1;
        }
    }
    return 0;
}


