# FilePaths plugin for MogileFS, by xb95 and hachi

#
# This plugin enables full pathing support within MogileFS, for creating files,
# listing files in a directory, deleting files, etc.
#
# Supports most functionality you'd expect.

package MogileFS::Plugin::FilePaths;

use strict;
use warnings;

our $VERSION = '0.02';
$VERSION = eval $VERSION;

use MogileFS::Worker::Query;
use MogileFS::Plugin::MetaData;

# called when this plugin is loaded, this sub must return a true value in order for
# MogileFS to consider the plugin to have loaded successfully.  if you return a
# non-true value, you MUST NOT install any handlers or other changes to the system.
# if you install something here, you MUST uninstall it in the unload sub.

sub _parse_path {
    my $fullpath = shift;
    return unless defined($fullpath) and length($fullpath);
    my ($path, $file) = $fullpath =~
        m!^(/(?:[\w\-\.]+/)*)([\w\-\.]+)$!;
    return ($path, $file);
}

sub load {

    # we want to remove the key being passed to create_open, as it is going to contain
    # only a path, and we want to ignore that for now
    MogileFS::register_global_hook( 'cmd_create_open', sub {
        my $args = shift;
        return 1 unless _check_dmid($args->{dmid});

        my $fullpath = delete $args->{key};
        my ($path, $filename) = _parse_path($fullpath);
        die "Filename is not a valid absolute path."
            unless defined($path) && length($path) && defined($filename) && length($filename);
        return 1;
    });

    # when people try to create new files, we need to intercept it and rewrite the
    # request a bit in order to do the right footwork to support paths.
    MogileFS::register_global_hook( 'cmd_create_close', sub {
        my $args = shift;
        return 1 unless _check_dmid($args->{dmid});

        # the key is the path, so we need to move that into the logical_path argument
        # and then set the key to be something more reasonable
        $args->{logical_path} = $args->{key};
        $args->{key} = "fid:$args->{fid}";
    });

    # called when we know a file has successfully been uploaded to the system, it's
    # a done deal, we don't have to worry about anything else
    MogileFS::register_global_hook( 'file_stored', sub {
        my $args = shift;
        return 1 unless _check_dmid($args->{dmid});

        # we need a path or this plugin is moot
        return 0 unless $args->{logical_path};

        # ensure we got a valid seeming path and filename
        my ($path, $filename) = _parse_path($args->{logical_path});
        return 0 unless defined($path) && length($path) && defined($filename) && length($filename);

        # great, let's vivify that path and get the node to it
        my $parentnodeid = MogileFS::Plugin::FilePaths::vivify_path( $args->{dmid}, $path );
        return 0 unless defined $parentnodeid;

        # see if this file exists already
        my $oldfid = MogileFS::Plugin::FilePaths::get_file_mapping( $args->{dmid}, $parentnodeid, $filename );
        if (defined $oldfid && $oldfid) {
            my $dbh = Mgd::get_dbh();
            $dbh->do("DELETE FROM file WHERE fid=?", undef, $oldfid);
            $dbh->do("REPLACE INTO file_to_delete SET fid=?", undef, $oldfid);
        }

        my $fid = $args->{fid};

        # and now, setup the mapping
        my $nodeid = MogileFS::Plugin::FilePaths::set_file_mapping( $args->{dmid}, $parentnodeid, $filename, $fid );
        return 0 unless $nodeid;

        if (my $keys = $args->{"plugin.meta.keys"}) {
            my %metadata;
            for (my $i = 0; $i < $keys; $i++) {
                my $key = $args->{"plugin.meta.key$i"};
                my $value = $args->{"plugin.meta.value$i"};
                $metadata{$key} = $value;
            }

            MogileFS::Plugin::MetaData::set_metadata($fid, \%metadata);
        }

        # we're successful, let's keep the file
        return 1;
    });

    # and now magic conversions that make the rest of the MogileFS commands work
    # without having to understand how the path system works
    MogileFS::register_global_hook( 'cmd_get_paths', \&_path_to_key );
    MogileFS::register_global_hook( 'cmd_delete', sub {
        my $args = shift;
        return 1 unless _check_dmid($args->{dmid});

        # ensure we got a valid seeming path and filename
        my ($path, $filename) = _parse_path($args->{key});
        return 0 unless defined($path) && length($path) && defined($filename) && length($filename);

        # now try to get the end of the path
        my $parentnodeid = MogileFS::Plugin::FilePaths::load_path( $args->{dmid}, $path );
        return 0 unless defined $parentnodeid;

        # get the fid of the file, bail out if it doesn't have one (directory nodes)
        my $fid = MogileFS::Plugin::FilePaths::get_file_mapping( $args->{dmid}, $parentnodeid, $filename );
        return 0 unless $fid;

        # great, delete this file
        delete_file_mapping( $args->{dmid}, $parentnodeid, $filename );
        # FIXME What should happen if this delete fails?

        # now pretend they asked for it and continue
        $args->{key} = "fid:$fid";
    });

    MogileFS::register_worker_command( 'filepaths_enable', sub {
        # get parameters
        my MogileFS::Worker::Query $self = shift;
        my $args = shift;

        # verify domain firstly
        my $dmid = $self->check_domain($args)
            or return $self->err_line('domain_not_found');

        my $dbh = Mgd::get_dbh();
        return undef unless $dbh;

        $dbh->do("REPLACE INTO plugin_filepaths_domains (dmid) VALUES (?)", undef, $dmid);

        return $self->err_line('unable_to_enable', "Unable to enable the filepaths plugin: " . $dbh->errstr)
            if $dbh->err;

        return $self->ok_line;
    });

    MogileFS::register_worker_command( 'filepaths_disable', sub {
        # get parameters
        my MogileFS::Worker::Query $self = shift;
        my $args = shift;

        # verify domain firstly
        my $dmid = $self->check_domain($args)
            or return $self->err_line('domain_not_found');

        my $dbh = Mgd::get_dbh();
        return undef unless $dbh;

        $dbh->do("DELETE FROM plugin_filepaths_domains WHERE dmid = ?", undef, $dmid);

        return $self->err_line('unable_to_disable', "Unable to enable the filepaths plugin: " . $dbh->errstr)
            if $dbh->err;

        return $self->ok_line;
    });

    # now let's define the extra plugin commands that we allow people to interact with us
    # just like with a regular MogileFS command
    MogileFS::register_worker_command( 'filepaths_list_directory', sub {
        # get parameters
        my MogileFS::Worker::Query $self = shift;
        my $args = shift;

        # verify domain firstly
        my $dmid = $self->check_domain($args)
            or return $self->err_line('domain_not_found');

        return $self->err_line("plugin_not_active_for_domain")
            unless _check_dmid($dmid);

        # verify arguments - only one expected, make sure it starts with a /
        my $path = $args->{arg1};
        return $self->err_line('bad_params')
            unless $args->{argcount} == 1 && $path && $path =~ /^\//;

        # now find the id of the path
        my $nodeid = MogileFS::Plugin::FilePaths::load_path( $dmid, $path );
        return $self->err_line('path_not_found', 'Path provided was not found in database')
            unless defined $nodeid;

#       TODO This is wrong, but we should throw an error saying 'not a directory'. Requires refactoring
#            a bit of code to make the 'fid' value available from the last node we fetched.
#        if (get_file_mapping($nodeid)) {
#            return $self->err_line('not_a_directory', 'Path provided is not a directory');
#        }

        # get files in path, return as an array
        my %res;
        my $ct = 0;
        my @nodes = MogileFS::Plugin::FilePaths::list_directory( $dmid, $nodeid );
        my $dbh = Mgd::get_dbh();

        my $node_count = $res{'files'} = scalar @nodes;

        for(my $i = 0; $i < $node_count; $i++) {
            my ($nodename, $fid) = @{$nodes[$i]};
            my $prefix = "file$i";
            $res{$prefix} = $nodename;

            if ($fid) { # This file is a regular file
                $res{"$prefix.type"} = "F";
                my $length = $dbh->selectrow_array("SELECT length FROM file WHERE fid=?", undef, $fid);
                $res{"$prefix.size"} = $length if defined($length);
                my $metadata = MogileFS::Plugin::MetaData::get_metadata($fid);
                $res{"$prefix.mtime"} = $metadata->{mtime} if $metadata->{mtime};
            } else {    # This file is a directory
                $res{"$prefix.type"} = "D";
            }
        }

        return $self->ok_line( \%res );
    });

    MogileFS::register_worker_command( 'filepaths_rename', sub {
        my MogileFS::Worker::Query $self = shift;
        my $args = shift;

        my $dmid = $self->check_domain($args)
            or return $self->err_line('domain_not_found');

        return $self->err_line("plugin_not_active_for_domain")
            unless _check_dmid($dmid);

        return $self->err_line("bad_argcount")
            unless $args->{argcount} == 2;

        my ($old_path, $old_name) = _parse_path($args->{arg1});

        return $self->err_line("badly_formed_orig")
            unless defined($old_path) && length($old_path) &&
                   defined($old_name) && length($old_name);

        my ($new_path, $new_name) = _parse_path($args->{arg2});

        return $self->err_line("badly_formed_new")
            unless defined($new_path) && length($new_path) &&
                   defined($new_name) && length($new_name);

        # I'd really like to lock on this operation at this point, but I find the whole idea to be rather
        # sad for the number of locks I would want to hold. Going to think about this instead and hope
        # nobody finds a way to make this race.

        # LOCK rename

        my $old_parentid = load_path($dmid, $old_path);
        my $new_parentid = vivify_path($dmid, $new_path);

        my $dbh = Mgd::get_dbh();
        return undef unless $dbh;

        $dbh->do('UPDATE plugin_filepaths_paths SET parentnodeid=?, nodename=? WHERE parentnodeid=? AND nodename=?', undef,
                 $new_parentid, $new_name, $old_parentid, $old_name);

        # UNLOCK rename

        return $self->err_line("rename_failed") if $dbh->err;

        return $self->ok_line();
    });

    return 1;
}

# this sub is called at the end or when the module is being unloaded, this needs to
# unregister any registered methods, etc.  you MUST uninstall everything that the
# plugin has previously installed.
sub unload {

    # remove our hooks
    MogileFS::unregister_global_hook( 'cmd_create_open' );
    MogileFS::unregister_global_hook( 'cmd_create_close' );
    MogileFS::unregister_global_hook( 'file_stored' );

    return 1;
}

# called when you want to create a path, this will break down the given argument and
# create any elements needed, returning the nodeid of the final node.  returns undef
# on error, else, 0-N is valid.
sub vivify_path {
    my ($dmid, $path) = @_;
    return undef unless $dmid && $path;
    return _traverse_path($dmid, $path, 1);
}

# called to load the nodeid of the final element in a path, which is useful for finding
# out if a path exists.  does NOT automatically create path elements that don't exist.
sub load_path {
    my ($dmid, $path) = @_;
    return undef unless $dmid && $path;
    return _traverse_path($dmid, $path, 0);
}

# does the internal work of traversing a path
sub _traverse_path {
    my ($dmid, $path, $vivify) = @_;
    return undef unless $dmid && $path;

    my @paths = grep { $_ } split /\//, $path;
    return 0 unless @paths; #toplevel

    # FIXME: validate_dbh()? or not needed? assumed done elsewhere? bleh.
    my $dbh = Mgd::get_dbh();
    return undef unless $dbh;

    my $parentnodeid = 0;
    foreach my $node (@paths) {
        # try to get the id for this node
        my $nodeid = _find_node($dbh, $dmid, $parentnodeid, $node, $vivify);
        return undef unless $nodeid;

        # this becomes the new parent
        $parentnodeid = $nodeid;
    }

    # we're done, so the parentnodeid is what we return
    return $parentnodeid;
}

# checks to see if a node exists, and if not, creates it if $vivify is set
sub _find_node {
    my ($dbh, $dmid, $parentnodeid, $node, $vivify) = @_;
    return undef unless $dbh && $dmid && defined $parentnodeid && $node;

    my $nodeid = $dbh->selectrow_array('SELECT nodeid FROM plugin_filepaths_paths ' .
                                       'WHERE dmid = ? AND parentnodeid = ? AND nodename = ?',
                                       undef, $dmid, $parentnodeid, $node);
    return undef if $dbh->err;
    return $nodeid if $nodeid;

    if ($vivify) {
        $dbh->do('INSERT INTO plugin_filepaths_paths (nodeid, dmid, parentnodeid, nodename, fid) ' .
                 'VALUES (NULL, ?, ?, ?, NULL)', undef, $dmid, $parentnodeid, $node);
        return undef if $dbh->err;

        $nodeid = $dbh->{mysql_insertid}+0;
    }

    return undef unless $nodeid && $nodeid > 0;
    return $nodeid;
}

# sets the mapping of a file from a name to a fid
sub set_file_mapping {
    my ($dmid, $parentnodeid, $filename, $fid) = @_;
    return undef unless $dmid && defined $parentnodeid && $filename && $fid;

    my $dbh = Mgd::get_dbh();
    return undef unless $dbh;

    my $nodeid = _find_node($dbh, $dmid, $parentnodeid, $filename, 1);
    return undef unless $nodeid;

    $dbh->do("UPDATE plugin_filepaths_paths SET fid = ? WHERE nodeid = ?", undef, $fid, $nodeid);
    return undef if $dbh->err;
    return $nodeid;
}

# given a domain and parent node and filename, return the fid
sub get_file_mapping {
    my ($dmid, $parentnodeid, $filename,) = @_;
    return undef unless $dmid && defined $parentnodeid && $filename;

    my $dbh = Mgd::get_dbh();
    return undef unless $dbh;

    my $fid = $dbh->selectrow_array('SELECT fid FROM plugin_filepaths_paths ' .
                                    'WHERE dmid = ? AND parentnodeid = ? AND nodename = ?',
                                    undef, $dmid, $parentnodeid, $filename);
    return undef if $dbh->err;
    return undef unless $fid && $fid > 0;
    return $fid;
}

sub delete_file_mapping {
    my ($dmid, $parentnodeid, $filename,) = @_;
    return undef unless $dmid && defined $parentnodeid && $filename;

    my $dbh = Mgd::get_dbh();
    return undef unless $dbh;

    $dbh->do('DELETE FROM plugin_filepaths_paths WHERE dmid = ? AND parentnodeid = ? AND nodename = ?',
             undef, $dmid, $parentnodeid, $filename);

    return undef if $dbh->err;
    return 1;
}

sub list_directory {
    my ($dmid, $nodeid) = @_;

    my $dbh = Mgd::get_dbh();
    return undef unless $dbh;

    my $sth = $dbh->prepare('SELECT nodename, fid FROM plugin_filepaths_paths ' .
                            'WHERE dmid = ? AND parentnodeid = ?');

    $sth->execute($dmid, $nodeid);

    my @return;

    while (my ($nodename, $fid) = $sth->fetchrow_array) {
        push @return, [$nodename, $fid];
    }

    return @return;
}

# generic sub that converts a file path to a key name that
# MogileFS will understand
sub _path_to_key {
    my $args = shift;

    my $dmid = $args->{dmid};
    return 1 unless _check_dmid($dmid);

    # ensure we got a valid seeming path and filename
    my ($path, $filename) =
        ($args->{key} =~ m!^(/(?:[\w\-\.]+/)*)([\w\-\.]+)$!) ? ($1, $2) : (undef, undef);
    return 0 unless $path && $filename;

    # now try to get the end of the path
    my $parentnodeid = MogileFS::Plugin::FilePaths::load_path( $dmid, $path );
    return 0 unless defined $parentnodeid;

    # great, find this file
    my $fid = MogileFS::Plugin::FilePaths::get_file_mapping( $dmid, $parentnodeid, $filename );
    return 0 unless defined $fid && $fid > 0;

    # now pretend they asked for it and continue
    $args->{key} = "fid:$fid";
    return 1;
}

my %active_dmids;
my $last_dmid_check = 0;

sub _check_dmid {
    my $dmid = shift;

    return unless defined $dmid;

    my $time = time();
    if ($time >= $last_dmid_check + 15) {
        $last_dmid_check = $time;

        unless (_load_dmids()) {
            warn "Unable to load active domains list for filepaths plugin, using old list";
        }
    }

    return $active_dmids{$dmid};
}

sub _load_dmids {
    my $dbh = Mgd::get_dbh();
    return undef unless $dbh;

    my $sth = $dbh->prepare('SELECT dmid FROM plugin_filepaths_domains');
    $sth->execute();

    return undef if $sth->err;

    %active_dmids = ();

    while (my $dmid = $sth->fetchrow_array) {
        $active_dmids{$dmid} = 1;
    }
    return 1;
}

package MogileFS::Store;

use MogileFS::Store;

use strict;
use warnings;

sub TABLE_plugin_filepaths_paths {
    "CREATE TABLE plugin_filepaths_paths (
        nodeid BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
        dmid SMALLINT UNSIGNED NOT NULL,
        parentnodeid BIGINT UNSIGNED NOT NULL,
        nodename VARCHAR(255) BINARY NOT NULL,
        fid BIGINT UNSIGNED,
        PRIMARY KEY (nodeid),
        UNIQUE KEY (dmid, parentnodeid, nodename)
)"
}

sub TABLE_plugin_filepaths_domains {
    "CREATE TABLE plugin_filepaths_domains (
        dmid SMALLINT UNSIGNED NOT NULL,
        PRIMARY KEY (dmid)
)"
}

__PACKAGE__->add_extra_tables("plugin_filepaths_paths", "plugin_filepaths_domains");

1;
