| Filename | /home/hinrik/perl5/perlbrew/perls/perl-5.13.5/lib/site_perl/5.13.5/Hailo/Storage.pm |
| Statements | Executed 66 statements in 2.40ms |
| Calls | P | F | Exclusive Time |
Inclusive Time |
Subroutine |
|---|---|---|---|---|---|
| 60001 | 2 | 2 | 88.5ms | 88.5ms | Hailo::Storage::_boundary_token_id (xsub) |
| 30002 | 3 | 3 | 52.0ms | 52.0ms | Hailo::Storage::_engaged (xsub) |
| 1 | 1 | 1 | 11.0ms | 14.8ms | Hailo::Storage::BEGIN@12 |
| 1 | 1 | 1 | 870µs | 1.06ms | Hailo::Storage::BEGIN@13 |
| 1 | 1 | 1 | 189µs | 781µs | Hailo::Storage::initialized |
| 1 | 1 | 1 | 106µs | 4.21ms | Hailo::Storage::_engage |
| 1 | 1 | 1 | 73µs | 231µs | Hailo::Storage::_engage_initialized_check_and_set_order |
| 1 | 1 | 1 | 67µs | 196µs | Hailo::Storage::_engage_initialized_check_and_set_tokenizer |
| 1 | 1 | 1 | 45µs | 130µs | Hailo::Storage::BEGIN@9 |
| 3 | 3 | 2 | 44µs | 2.74ms | Hailo::Storage::sth (xsub) |
| 8 | 7 | 2 | 42µs | 5.81ms | Hailo::Storage::dbh (xsub) |
| 1 | 1 | 1 | 28µs | 2.69ms | Hailo::Storage::_build_sth |
| 1 | 1 | 1 | 28µs | 115µs | Hailo::Storage::_build_dbi_options |
| 1 | 1 | 1 | 24µs | 5.76ms | Hailo::Storage::_build_dbh |
| 1 | 1 | 1 | 22µs | 22µs | Hailo::Storage::BEGIN@2 |
| 2 | 2 | 1 | 20µs | 26µs | Hailo::Storage::dbd (xsub) |
| 2 | 2 | 1 | 18µs | 213µs | Hailo::Storage::dbi_options (xsub) |
| 1 | 1 | 1 | 17µs | 63µs | Hailo::Storage::dbd_options (xsub) |
| 1 | 1 | 1 | 12µs | 84µs | Hailo::Storage::BEGIN@9.6 |
| 1 | 1 | 1 | 11µs | 353µs | Hailo::Storage::BEGIN@11 |
| 1 | 1 | 1 | 11µs | 645µs | Hailo::Storage::BEGIN@10 |
| 1 | 1 | 1 | 9µs | 9µs | Hailo::Storage::_build_dbd_options |
| 1 | 1 | 1 | 6µs | 6µs | Hailo::Storage::BEGIN@5 |
| 0 | 0 | 0 | 0s | 0s | Hailo::Storage::__ANON__[:244] |
| 0 | 0 | 0 | 0s | 0s | Hailo::Storage::start_learning |
| 0 | 0 | 0 | 0s | 0s | Hailo::Storage::start_training |
| 0 | 0 | 0 | 0s | 0s | Hailo::Storage::stop_learning |
| 0 | 0 | 0 | 0s | 0s | Hailo::Storage::stop_training |
| 0 | 0 | 0 | 0s | 0s | Hailo::Storage::totals |
| Line | State ments |
Time on line |
Calls | Time in subs |
Code |
|---|---|---|---|---|---|
| 1 | package Hailo::Storage; | ||||
| 2 | # spent 22µs within Hailo::Storage::BEGIN@2 which was called:
# once (22µs+0s) by Mouse::Util::_try_load_one_class at line 4 | ||||
| 3 | 1 | 7µs | $Hailo::Storage::AUTHORITY = 'cpan:AVAR'; | ||
| 4 | 1 | 24µs | 1 | 22µs | } # spent 22µs making 1 call to Hailo::Storage::BEGIN@2 |
| 5 | # spent 6µs within Hailo::Storage::BEGIN@5 which was called:
# once (6µs+0s) by Mouse::Util::_try_load_one_class at line 7 | ||||
| 6 | 1 | 6µs | $Hailo::Storage::VERSION = '0.57'; | ||
| 7 | 1 | 26µs | 1 | 6µs | } # spent 6µs making 1 call to Hailo::Storage::BEGIN@5 |
| 8 | |||||
| 9 | 4 | 73µs | 3 | 286µs | use 5.010; # spent 130µs making 1 call to Hailo::Storage::BEGIN@9
# spent 84µs making 1 call to Hailo::Storage::BEGIN@9.6
# spent 72µs making 1 call to feature::import |
| 10 | 2 | 31µs | 2 | 1.28ms | # spent 645µs (11+634) within Hailo::Storage::BEGIN@10 which was called:
# once (11µs+634µs) by Mouse::Util::_try_load_one_class at line 10 # spent 645µs making 1 call to Hailo::Storage::BEGIN@10
# spent 634µs making 1 call to Any::Moose::import |
| 11 | 2 | 29µs | 2 | 694µs | # spent 353µs (11+341) within Hailo::Storage::BEGIN@11 which was called:
# once (11µs+341µs) by Mouse::Util::_try_load_one_class at line 11 # spent 353µs making 1 call to Hailo::Storage::BEGIN@11
# spent 342µs making 1 call to Any::Moose::import |
| 12 | 2 | 152µs | 2 | 14.9ms | # spent 14.8ms (11.0+3.82) within Hailo::Storage::BEGIN@12 which was called:
# once (11.0ms+3.82ms) by Mouse::Util::_try_load_one_class at line 12 # spent 14.8ms making 1 call to Hailo::Storage::BEGIN@12
# spent 48µs making 1 call to Exporter::import |
| 13 | 2 | 1.19ms | 1 | 1.06ms | # spent 1.06ms (870µs+191µs) within Hailo::Storage::BEGIN@13 which was called:
# once (870µs+191µs) by Mouse::Util::_try_load_one_class at line 13 # spent 1.06ms making 1 call to Hailo::Storage::BEGIN@13 |
| 14 | |||||
| 15 | 1 | 6µs | 1 | 425µs | has dbd => ( # spent 425µs making 1 call to Mouse::has |
| 16 | isa => 'Str', | ||||
| 17 | is => 'ro', | ||||
| 18 | lazy_build => 1, | ||||
| 19 | documentation => "The DBD::* driver we're using", | ||||
| 20 | ); | ||||
| 21 | |||||
| 22 | 1 | 4µs | 1 | 333µs | has dbd_options => ( # spent 333µs making 1 call to Mouse::has |
| 23 | isa => 'HashRef', | ||||
| 24 | is => 'ro', | ||||
| 25 | lazy_build => 1, | ||||
| 26 | documentation => 'Options passed as the last argument to DBI->connect()', | ||||
| 27 | ); | ||||
| 28 | |||||
| 29 | # spent 9µs within Hailo::Storage::_build_dbd_options which was called:
# once (9µs+0s) by Mouse::super at line 93 of Mouse.pm | ||||
| 30 | 2 | 8µs | my ($self) = @_; | ||
| 31 | return { | ||||
| 32 | RaiseError => 1 | ||||
| 33 | }; | ||||
| 34 | } | ||||
| 35 | |||||
| 36 | 1 | 4µs | 1 | 596µs | has dbh => ( # spent 596µs making 1 call to Mouse::has |
| 37 | isa => 'DBI::db', | ||||
| 38 | is => 'ro', | ||||
| 39 | lazy_build => 1, | ||||
| 40 | documentation => 'Our DBD object', | ||||
| 41 | ); | ||||
| 42 | |||||
| 43 | # spent 5.76ms (24µs+5.73) within Hailo::Storage::_build_dbh which was called:
# once (24µs+5.73ms) by Hailo::Storage::dbh at line 115 of Hailo/Storage/SQLite.pm | ||||
| 44 | 3 | 30µs | my ($self) = @_; | ||
| 45 | 1 | 12µs | 4 | 406µs | my $dbd_options = $self->dbi_options; # spent 211µs making 1 call to Hailo::Storage::dbi_options
# spent 192µs making 1 call to Hailo::Storage::SQLite::_build_dbi_options
# spent 2µs making 1 call to Mouse::Meta::TypeConstraint::_compiled_type_constraint
# spent 1µs making 1 call to Mouse::Meta::Attribute::builder |
| 46 | |||||
| 47 | 2 | 5.52ms | return DBI->connect($self->dbi_options); # spent 5.52ms making 1 call to DBI::connect
# spent 2µs making 1 call to Hailo::Storage::dbi_options | ||
| 48 | }; | ||||
| 49 | |||||
| 50 | 1 | 4µs | 1 | 386µs | has dbi_options => ( # spent 386µs making 1 call to Mouse::has |
| 51 | isa => 'ArrayRef', | ||||
| 52 | is => 'ro', | ||||
| 53 | auto_deref => 1, | ||||
| 54 | lazy_build => 1, | ||||
| 55 | documentation => 'Options passed to DBI->connect()', | ||||
| 56 | ); | ||||
| 57 | |||||
| 58 | # spent 115µs (28+88) within Hailo::Storage::_build_dbi_options which was called:
# once (28µs+88µs) by Hailo::Storage::SQLite::__ANON__[/home/hinrik/perl5/perlbrew/perls/perl-5.13.5/lib/site_perl/5.13.5/Hailo/Storage/SQLite.pm:42] at line 34 of Hailo/Storage/SQLite.pm | ||||
| 59 | 6 | 42µs | my ($self) = @_; | ||
| 60 | 1 | 12µs | 4 | 29µs | my $dbd = $self->dbd; # spent 23µs making 1 call to Hailo::Storage::dbd
# spent 3µs making 1 call to Hailo::Storage::SQLite::_build_dbd
# spent 2µs making 1 call to Mouse::Meta::TypeConstraint::_compiled_type_constraint
# spent 1µs making 1 call to Mouse::Meta::Attribute::builder |
| 61 | 1 | 13µs | 4 | 109µs | my $dbd_options = $self->dbd_options; # spent 63µs making 1 call to Hailo::Storage::dbd_options
# spent 43µs making 1 call to Hailo::Storage::SQLite::_build_dbd_options
# spent 2µs making 1 call to Mouse::Meta::Attribute::builder
# spent 1µs making 1 call to Mouse::Meta::TypeConstraint::_compiled_type_constraint |
| 62 | 1 | 1µs | my $db = $self->brain // ''; # spent 1µs making 1 call to Hailo::Storage::SQLite::brain | ||
| 63 | |||||
| 64 | my @options = ( | ||||
| 65 | "dbi:$dbd:dbname=$db", | ||||
| 66 | '', | ||||
| 67 | '', | ||||
| 68 | $dbd_options, | ||||
| 69 | ); | ||||
| 70 | |||||
| 71 | return \@options; | ||||
| 72 | } | ||||
| 73 | |||||
| 74 | 1 | 4µs | 1 | 262µs | has _engaged => ( # spent 262µs making 1 call to Mouse::has |
| 75 | isa => 'Bool', | ||||
| 76 | is => 'rw', | ||||
| 77 | default => 0, | ||||
| 78 | documentation => 'Have we done setup work to get this database going?', | ||||
| 79 | ); | ||||
| 80 | |||||
| 81 | 1 | 4µs | 1 | 342µs | has sth => ( # spent 342µs making 1 call to Mouse::has |
| 82 | isa => 'HashRef', | ||||
| 83 | is => 'ro', | ||||
| 84 | lazy_build => 1, | ||||
| 85 | documentation => 'A HashRef of prepared DBI statement handles', | ||||
| 86 | ); | ||||
| 87 | |||||
| 88 | # spent 2.69ms (28µs+2.66) within Hailo::Storage::_build_sth which was called:
# once (28µs+2.66ms) by Hailo::Storage::sth at line 110 | ||||
| 89 | 2 | 32µs | my ($self) = @_; | ||
| 90 | 4 | 2.66ms | return Hailo::Storage::Schema->sth($self->dbd, $self->dbh, $self->order); # spent 2.66ms making 1 call to Hailo::Storage::Schema::sth
# spent 2µs making 1 call to Hailo::Storage::dbd
# spent 2µs making 1 call to Hailo::Storage::SQLite::order
# spent 1µs making 1 call to Hailo::Storage::dbh | ||
| 91 | } | ||||
| 92 | |||||
| 93 | 1 | 4µs | 1 | 259µs | has _boundary_token_id => ( # spent 259µs making 1 call to Mouse::has |
| 94 | isa => 'Int', | ||||
| 95 | is => 'rw', | ||||
| 96 | ); | ||||
| 97 | |||||
| 98 | # bootstrap the database | ||||
| 99 | # spent 4.21ms (106µs+4.10) within Hailo::Storage::_engage which was called:
# once (106µs+4.10ms) by Hailo::Storage::SQLite::_engage at line 332 of Mouse/Meta/Class.pm | ||||
| 100 | 4 | 28µs | my ($self) = @_; | ||
| 101 | |||||
| 102 | 5 | 113µs | 1 | 881µs | if ($self->initialized) { # spent 881µs making 1 call to Hailo::Storage::SQLite::initialized |
| 103 | # Check the order we've been given and retrieve it from the | ||||
| 104 | # database if there's nothing odd going on. | ||||
| 105 | 4 | 240µs | $self->_engage_initialized_check_and_set_order; # spent 231µs making 1 call to Hailo::Storage::_engage_initialized_check_and_set_order
# spent 7µs making 2 calls to DBI::common::DESTROY, avg 4µs/call
# spent 2µs making 1 call to DBD::_mem::common::DESTROY | ||
| 106 | |||||
| 107 | # Likewise for the Tokenizer | ||||
| 108 | 4 | 206µs | $self->_engage_initialized_check_and_set_tokenizer; # spent 196µs making 1 call to Hailo::Storage::_engage_initialized_check_and_set_tokenizer
# spent 8µs making 2 calls to DBI::common::DESTROY, avg 4µs/call
# spent 2µs making 1 call to DBD::_mem::common::DESTROY | ||
| 109 | |||||
| 110 | 1 | 56µs | 5 | 5.46ms | $self->sth->{token_id}->execute(0, ''); # spent 2.74ms making 1 call to Hailo::Storage::sth
# spent 2.69ms making 1 call to Hailo::Storage::_build_sth
# spent 24µs making 1 call to DBI::st::execute
# spent 3µs making 1 call to Mouse::Meta::TypeConstraint::_compiled_type_constraint
# spent 2µs making 1 call to Mouse::Meta::Attribute::builder |
| 111 | 2 | 13µs | my $id = $self->sth->{token_id}->fetchrow_array; # spent 11µs making 1 call to DBI::st::fetchrow_array
# spent 2µs making 1 call to Hailo::Storage::sth | ||
| 112 | 2 | 17µs | $self->_boundary_token_id($id); # spent 15µs making 1 call to Hailo::Storage::_boundary_token_id
# spent 2µs making 1 call to Mouse::Meta::TypeConstraint::_compiled_type_constraint | ||
| 113 | } | ||||
| 114 | else { | ||||
| 115 | Hailo::Storage::Schema->deploy($self->dbd, $self->dbh, $self->order); | ||||
| 116 | |||||
| 117 | # Set metadata in the database for use by subsequent | ||||
| 118 | # invocations | ||||
| 119 | { | ||||
| 120 | # Don't change order again | ||||
| 121 | my $order = $self->order; | ||||
| 122 | $self->sth->{set_info}->execute('markov_order', $order); | ||||
| 123 | |||||
| 124 | # Warn if the tokenizer changes | ||||
| 125 | my $tokenizer = $self->tokenizer_class; | ||||
| 126 | $self->sth->{set_info}->execute('tokenizer_class', $tokenizer); | ||||
| 127 | } | ||||
| 128 | |||||
| 129 | $self->sth->{add_token}->execute(0, ''); | ||||
| 130 | $self->sth->{last_token_rowid}->execute(); | ||||
| 131 | my $id = $self->sth->{last_token_rowid}->fetchrow_array(); | ||||
| 132 | $self->_boundary_token_id($id); | ||||
| 133 | } | ||||
| 134 | |||||
| 135 | 1 | 3µs | $self->_engaged(1); # spent 3µs making 1 call to Hailo::Storage::_engaged | ||
| 136 | |||||
| 137 | return; | ||||
| 138 | } | ||||
| 139 | |||||
| 140 | # spent 231µs (73+157) within Hailo::Storage::_engage_initialized_check_and_set_order which was called:
# once (73µs+157µs) by Hailo::Storage::_engage at line 105 | ||||
| 141 | 7 | 132µs | my ($self) = @_; | ||
| 142 | |||||
| 143 | 1 | 7µs | 3 | 192µs | my $sth = $self->dbh->prepare(qq[SELECT text FROM info WHERE attribute = ?;]); # spent 100µs making 1 call to DBI::db::prepare
# spent 90µs making 1 call to DBD::SQLite::db::prepare
# spent 2µs making 1 call to Hailo::Storage::dbh |
| 144 | 1 | 24µs | $sth->execute('markov_order'); # spent 24µs making 1 call to DBI::st::execute | ||
| 145 | 1 | 19µs | my $db_order = $sth->fetchrow_array(); # spent 19µs making 1 call to DBI::st::fetchrow_array | ||
| 146 | |||||
| 147 | 1 | 3µs | my $my_order = $self->order; # spent 3µs making 1 call to Hailo::Storage::SQLite::order | ||
| 148 | if ($my_order != $db_order) { | ||||
| 149 | if ($self->hailo->{has_custom_order}->()) { | ||||
| 150 | die <<"DIE"; | ||||
| 151 | You've manually supplied an order of `$my_order' to Hailo but you're | ||||
| 152 | loading a brain that has the order `$db_order'. | ||||
| 153 | |||||
| 154 | Hailo will automatically load the order from existing brains, however | ||||
| 155 | you've constructed Hailo and manually specified an order not | ||||
| 156 | equivalent to the existing order of the database. | ||||
| 157 | |||||
| 158 | Either supply the correct order or omit the order attribute | ||||
| 159 | altogether. We could continue but I'd rather die since you're probably | ||||
| 160 | expecting something I can't deliver. | ||||
| 161 | DIE | ||||
| 162 | } | ||||
| 163 | |||||
| 164 | $self->order($db_order); | ||||
| 165 | $self->hailo->{set_order}->($db_order); | ||||
| 166 | } | ||||
| 167 | |||||
| 168 | return; | ||||
| 169 | } | ||||
| 170 | |||||
| 171 | # spent 196µs (67+130) within Hailo::Storage::_engage_initialized_check_and_set_tokenizer which was called:
# once (67µs+130µs) by Hailo::Storage::_engage at line 108 | ||||
| 172 | 7 | 107µs | my ($self) = @_; | ||
| 173 | |||||
| 174 | 1 | 7µs | 3 | 176µs | my $sth = $self->dbh->prepare(qq[SELECT text FROM info WHERE attribute = ?;]); # spent 92µs making 1 call to DBI::db::prepare
# spent 82µs making 1 call to DBD::SQLite::db::prepare
# spent 2µs making 1 call to Hailo::Storage::dbh |
| 175 | 1 | 15µs | $sth->execute('tokenizer_class'); # spent 15µs making 1 call to DBI::st::execute | ||
| 176 | 1 | 8µs | my $db_tokenizer_class = $sth->fetchrow_array; # spent 8µs making 1 call to DBI::st::fetchrow_array | ||
| 177 | 1 | 4µs | my $my_tokenizer_class = $self->tokenizer_class; # spent 4µs making 1 call to Hailo::Storage::SQLite::tokenizer_class | ||
| 178 | |||||
| 179 | # defined() because we can't count on old brains having this | ||||
| 180 | if (defined $db_tokenizer_class | ||||
| 181 | and $my_tokenizer_class ne $db_tokenizer_class) { | ||||
| 182 | if ($self->hailo->{has_custom_tokenizer_class}->()) { | ||||
| 183 | die <<"DIE"; | ||||
| 184 | You've manually supplied a tokenizer class `$my_tokenizer_class' to | ||||
| 185 | Hailo, but you're loading a brain that has the tokenizer class | ||||
| 186 | `$db_tokenizer_class'. | ||||
| 187 | |||||
| 188 | Hailo will automatically load the tokenizer class from existing | ||||
| 189 | brains, however you've constructed Hailo and manually specified an | ||||
| 190 | tokenizer class not equivalent to the existing tokenizer class of the | ||||
| 191 | database. | ||||
| 192 | |||||
| 193 | Either supply the correct tokenizer class or omit the order attribute | ||||
| 194 | altogether. We could continue but I'd rather die since you're probably | ||||
| 195 | expecting something I can't deliver. | ||||
| 196 | DIE | ||||
| 197 | } | ||||
| 198 | |||||
| 199 | $self->tokenizer_class($db_tokenizer_class); | ||||
| 200 | $self->hailo->{set_tokenizer_class}->($db_tokenizer_class); | ||||
| 201 | } | ||||
| 202 | |||||
| 203 | return; | ||||
| 204 | } | ||||
| 205 | |||||
| 206 | sub start_training { | ||||
| 207 | my ($self) = @_; | ||||
| 208 | $self->_engage() unless $self->_engaged; | ||||
| 209 | $self->start_learning(); | ||||
| 210 | return; | ||||
| 211 | } | ||||
| 212 | |||||
| 213 | sub stop_training { | ||||
| 214 | my ($self) = @_; | ||||
| 215 | $self->stop_learning(); | ||||
| 216 | return; | ||||
| 217 | } | ||||
| 218 | |||||
| 219 | sub start_learning { | ||||
| 220 | my ($self) = @_; | ||||
| 221 | $self->_engage() unless $self->_engaged; | ||||
| 222 | |||||
| 223 | # start a transaction | ||||
| 224 | $self->dbh->begin_work; | ||||
| 225 | return; | ||||
| 226 | } | ||||
| 227 | |||||
| 228 | sub stop_learning { | ||||
| 229 | my ($self) = @_; | ||||
| 230 | # finish a transaction | ||||
| 231 | $self->dbh->commit; | ||||
| 232 | return; | ||||
| 233 | } | ||||
| 234 | |||||
| 235 | # See if SELECT count(*) FROM info; fails. If not we assume that we | ||||
| 236 | # have an up and running database. | ||||
| 237 | # spent 781µs (189+591) within Hailo::Storage::initialized which was called:
# once (189µs+591µs) by Mouse::super at line 93 of Mouse.pm | ||||
| 238 | 5 | 27µs | my ($self) = @_; | ||
| 239 | 1 | 2µs | my $dbh = $self->dbh; # spent 2µs making 1 call to Hailo::Storage::dbh | ||
| 240 | |||||
| 241 | my ($err, $warn, $res); | ||||
| 242 | 3 | 170µs | eval { | ||
| 243 | # SQLite will warn 'no such table info' | ||||
| 244 | local $SIG{__WARN__} = sub { $err = $_[0] }; | ||||
| 245 | |||||
| 246 | # If it doesn't warn trust that it dies here | ||||
| 247 | local ($@, $!); | ||||
| 248 | 1 | 14µs | 2 | 1.16ms | $res = $dbh->do("SELECT count(*) FROM info;"); # spent 589µs making 1 call to DBI::db::do
# spent 572µs making 1 call to DBD::SQLite::db::do |
| 249 | }; | ||||
| 250 | |||||
| 251 | return (not $err and not $warn and defined $res); | ||||
| 252 | } | ||||
| 253 | |||||
| 254 | # return some statistics | ||||
| 255 | sub totals { | ||||
| 256 | my ($self) = @_; | ||||
| 257 | $self->_engage() unless $self->_engaged; | ||||
| 258 | |||||
| 259 | $self->sth->{token_total}->execute(); | ||||
| 260 | my $token = $self->sth->{token_total}->fetchrow_array - 1; | ||||
| 261 | $self->sth->{expr_total}->execute(); | ||||
| 262 | my $expr = $self->sth->{expr_total}->fetchrow_array // 0; | ||||
| 263 | $self->sth->{prev_total}->execute(); | ||||
| 264 | my $prev = $self->sth->{prev_total}->fetchrow_array // 0; | ||||
| 265 | $self->sth->{next_total}->execute(); | ||||
| 266 | my $next = $self->sth->{next_total}->fetchrow_array // 0; | ||||
| 267 | |||||
| 268 | return $token, $expr, $prev, $next; | ||||
| 269 | } | ||||
| 270 | |||||
| 271 | 1 | 21µs | 2 | 142µs | __PACKAGE__->meta->make_immutable; # spent 124µs making 1 call to Mouse::Meta::Class::make_immutable
# spent 18µs making 1 call to Hailo::Storage::meta |
| 272 | |||||
| 273 | =encoding utf8 | ||||
| 274 | |||||
| 275 | =head1 NAME | ||||
| 276 | |||||
| 277 | Hailo::Storage - A base class for L<Hailo> L<storage|Hailo::Role::Storage> backends | ||||
| 278 | |||||
| 279 | =head1 METHODS | ||||
| 280 | |||||
| 281 | The following methods must to be implemented by subclasses: | ||||
| 282 | |||||
| 283 | =head2 C<_build_dbd> | ||||
| 284 | |||||
| 285 | Should return the name of the database driver (e.g. 'SQLite') which will be | ||||
| 286 | passed to L<DBI|DBI>. | ||||
| 287 | |||||
| 288 | =head2 C<_build_dbd_options> | ||||
| 289 | |||||
| 290 | Subclasses can override this method to add options of their own. E.g: | ||||
| 291 | |||||
| 292 | override _build_dbd_options => sub { | ||||
| 293 | return { | ||||
| 294 | %{ super() }, | ||||
| 295 | sqlite_unicode => 1, | ||||
| 296 | }; | ||||
| 297 | }; | ||||
| 298 | |||||
| 299 | =head2 C<initialized> | ||||
| 300 | |||||
| 301 | Should return a true value if the database has already been created. | ||||
| 302 | |||||
| 303 | =head1 Comparison of backends | ||||
| 304 | |||||
| 305 | This benchmark shows how the backends compare when training on the | ||||
| 306 | small testsuite dataset as reported by the F<utils/hailo-benchmark> | ||||
| 307 | utility (found in the distribution): | ||||
| 308 | |||||
| 309 | Rate DBD::Pg DBD::mysql DBD::SQLite/file DBD::SQLite/memory | ||||
| 310 | DBD::Pg 2.22/s -- -33% -49% -56% | ||||
| 311 | DBD::mysql 3.33/s 50% -- -23% -33% | ||||
| 312 | DBD::SQLite/file 4.35/s 96% 30% -- -13% | ||||
| 313 | DBD::SQLite/memory 5.00/s 125% 50% 15% -- | ||||
| 314 | |||||
| 315 | Under real-world workloads SQLite is much faster than these results | ||||
| 316 | indicate since the time it takes to train/reply is relative to the | ||||
| 317 | existing database size. Here's how long it took to train on a 214,710 | ||||
| 318 | line IRC log on a Linode 1080 with Hailo 0.18: | ||||
| 319 | |||||
| 320 | =over | ||||
| 321 | |||||
| 322 | =item * SQLite | ||||
| 323 | |||||
| 324 | real 8m38.285s | ||||
| 325 | user 8m30.831s | ||||
| 326 | sys 0m1.175s | ||||
| 327 | |||||
| 328 | =item * MySQL | ||||
| 329 | |||||
| 330 | real 48m30.334s | ||||
| 331 | user 8m25.414s | ||||
| 332 | sys 4m38.175s | ||||
| 333 | |||||
| 334 | =item * PostgreSQL | ||||
| 335 | |||||
| 336 | real 216m38.906s | ||||
| 337 | user 11m13.474s | ||||
| 338 | sys 4m35.509s | ||||
| 339 | |||||
| 340 | =back | ||||
| 341 | |||||
| 342 | In the case of PostgreSQL it's actually much faster to first train | ||||
| 343 | with SQLite, dump that database and then import it with L<psql(1)>, | ||||
| 344 | see L<failo's README|http://github.com/hinrik/failo> for how to do | ||||
| 345 | that. | ||||
| 346 | |||||
| 347 | However when replying with an existing database (using | ||||
| 348 | F<utils/hailo-benchmark-replies>) yields different results. SQLite can | ||||
| 349 | reply really quickly without being warmed up (which is the typical | ||||
| 350 | usecase for chatbots) but once PostgreSQL and MySQL are warmed up they | ||||
| 351 | start replying faster: | ||||
| 352 | |||||
| 353 | Here's a comparison of doing 10 replies: | ||||
| 354 | |||||
| 355 | Rate PostgreSQL MySQL SQLite-file SQLite-file-28MB SQLite-memory | ||||
| 356 | PostgreSQL 71.4/s -- -14% -14% -29% -50% | ||||
| 357 | MySQL 83.3/s 17% -- 0% -17% -42% | ||||
| 358 | SQLite-file 83.3/s 17% 0% -- -17% -42% | ||||
| 359 | SQLite-file-28MB 100.0/s 40% 20% 20% -- -30% | ||||
| 360 | SQLite-memory 143/s 100% 71% 71% 43% -- | ||||
| 361 | |||||
| 362 | In this test MySQL uses around 28MB of memory (using Debian's | ||||
| 363 | F<my-small.cnf>) and PostgreSQL around 34MB. Plain SQLite uses 2MB of | ||||
| 364 | cache but it's also tested with 28MB of cache as well as with the | ||||
| 365 | entire database in memory. | ||||
| 366 | |||||
| 367 | But doing 10,000 replies is very different: | ||||
| 368 | |||||
| 369 | Rate SQLite-file PostgreSQL SQLite-file-28MB MySQL SQLite-memory | ||||
| 370 | SQLite-file 85.1/s -- -7% -18% -27% -38% | ||||
| 371 | PostgreSQL 91.4/s 7% -- -12% -21% -33% | ||||
| 372 | SQLite-file-28MB 103/s 21% 13% -- -11% -25% | ||||
| 373 | MySQL 116/s 37% 27% 13% -- -15% | ||||
| 374 | SQLite-memory 137/s 61% 50% 33% 18% -- | ||||
| 375 | |||||
| 376 | Once MySQL gets more memory (using Debian's F<my-large.cnf>) and a | ||||
| 377 | chance to warm it starts yielding better results (I couldn't find out | ||||
| 378 | how to make PostgreSQL take as much memory as it wanted): | ||||
| 379 | |||||
| 380 | Rate MySQL SQLite-memory | ||||
| 381 | MySQL 121/s -- -12% | ||||
| 382 | SQLite-memory 138/s 14% -- | ||||
| 383 | |||||
| 384 | =head1 AUTHOR | ||||
| 385 | |||||
| 386 | E<AElig>var ArnfjE<ouml>rE<eth> Bjarmason <avar@cpan.org> | ||||
| 387 | |||||
| 388 | Hinrik E<Ouml>rn SigurE<eth>sson, hinrik.sig@gmail.com | ||||
| 389 | |||||
| 390 | =head1 LICENSE AND COPYRIGHT | ||||
| 391 | |||||
| 392 | Copyright 2010 E<AElig>var ArnfjE<ouml>rE<eth> Bjarmason and | ||||
| 393 | Hinrik E<Ouml>rn SigurE<eth>sson | ||||
| 394 | |||||
| 395 | This program is free software, you can redistribute it and/or modify | ||||
| 396 | it under the same terms as Perl itself. | ||||
| 397 | |||||
| 398 | =cut | ||||
# spent 88.5ms (88.5+2µs) within Hailo::Storage::_boundary_token_id which was called 60001 times, avg 1µs/call:
# 60000 times (88.5ms+0s) by Hailo::Engine::Default::_construct_reply at line 288 of Hailo/Engine/Default.pm, avg 1µs/call
# once (13µs+2µs) by Hailo::Storage::_engage at line 112 | |||||
# spent 52.0ms within Hailo::Storage::_engaged which was called 30002 times, avg 2µs/call:
# 30000 times (52.0ms+0s) by Hailo::reply at line 314 of Hailo.pm, avg 2µs/call
# once (3µs+0s) by Hailo::Storage::_engage at line 135
# once (2µs+0s) by Hailo::Storage::SQLite::save at line 126 of Hailo/Storage/SQLite.pm | |||||
sub Hailo::Storage::dbd; # xsub | |||||
# spent 63µs (17+46) within Hailo::Storage::dbd_options which was called:
# once (17µs+46µs) by Hailo::Storage::_build_dbi_options at line 61 | |||||
# spent 5.81ms (42µs+5.76) within Hailo::Storage::dbh which was called 8 times, avg 726µs/call:
# 2 times (31µs+5.76ms) by Hailo::Storage::SQLite::_set_pragmas at line 115 of Hailo/Storage/SQLite.pm, avg 2.90ms/call
# once (2µs+0s) by Hailo::Storage::initialized at line 239
# once (2µs+0s) by Hailo::Storage::_engage_initialized_check_and_set_order at line 143
# once (2µs+0s) by Hailo::Storage::SQLite::save at line 127 of Hailo/Storage/SQLite.pm
# once (2µs+0s) by Hailo::Storage::SQLite::__ANON__[/home/hinrik/perl5/perlbrew/perls/perl-5.13.5/lib/site_perl/5.13.5/Hailo/Storage/SQLite.pm:65] at line 60 of Hailo/Storage/SQLite.pm
# once (2µs+0s) by Hailo::Storage::_engage_initialized_check_and_set_tokenizer at line 174
# once (1µs+0s) by Hailo::Storage::_build_sth at line 90 | |||||
sub Hailo::Storage::dbi_options; # xsub | |||||
# spent 2.74ms (44µs+2.70) within Hailo::Storage::sth which was called 3 times, avg 914µs/call:
# once (41µs+2.70ms) by Hailo::Storage::_engage at line 110
# once (2µs+0s) by Hailo::Storage::_engage at line 111
# once (2µs+0s) by Hailo::Engine::Default::BUILD at line 35 of Hailo/Engine/Default.pm |