| Filename | /mnt/stuff/src/my-cpan/hailo/lib/Hailo/Storage.pm |
| Statements | Executed 70 statements in 6.15ms |
| Calls | P | F | Exclusive Time |
Inclusive Time |
Subroutine |
|---|---|---|---|---|---|
| 131075 | 2 | 2 | 200ms | 200ms | Hailo::Storage::_boundary_token_id (xsub) |
| 98607 | 10 | 3 | 133ms | 180ms | Hailo::Storage::dbh (xsub) |
| 1 | 1 | 1 | 11.1ms | 13.4ms | Hailo::Storage::BEGIN@6 |
| 1 | 1 | 1 | 706µs | 875µs | Hailo::Storage::BEGIN@7 |
| 1 | 1 | 1 | 120µs | 259µs | Hailo::Storage::initialized |
| 1 | 1 | 1 | 115µs | 5.18ms | Hailo::Storage::_engage |
| 1 | 1 | 1 | 56µs | 141µs | Hailo::Storage::BEGIN@3 |
| 1 | 1 | 1 | 42µs | 4.05ms | Hailo::Storage::stop_learning |
| 6 | 6 | 2 | 34µs | 1.97ms | Hailo::Storage::sth (xsub) |
| 1 | 1 | 1 | 28µs | 5.61ms | Hailo::Storage::start_training |
| 1 | 1 | 1 | 28µs | 114µs | Hailo::Storage::_build_dbi_options |
| 3 | 3 | 1 | 24µs | 30µs | Hailo::Storage::dbd (xsub) |
| 1 | 1 | 1 | 22µs | 46.9ms | Hailo::Storage::_build_dbh |
| 1 | 1 | 1 | 19µs | 1.94ms | Hailo::Storage::_build_sth |
| 1 | 1 | 1 | 19µs | 4.07ms | Hailo::Storage::stop_training |
| 1 | 1 | 1 | 18µs | 66µs | Hailo::Storage::start_learning |
| 1 | 1 | 1 | 18µs | 59µs | Hailo::Storage::dbd_options (xsub) |
| 2 | 2 | 1 | 17µs | 197µs | Hailo::Storage::dbi_options (xsub) |
| 1 | 1 | 1 | 13µs | 599µs | Hailo::Storage::BEGIN@4 |
| 1 | 1 | 1 | 12µs | 86µs | Hailo::Storage::BEGIN@3.12 |
| 1 | 1 | 1 | 11µs | 339µs | Hailo::Storage::BEGIN@5 |
| 4 | 4 | 2 | 8µs | 8µs | Hailo::Storage::_engaged (xsub) |
| 1 | 1 | 1 | 5µs | 5µs | Hailo::Storage::_build_dbd_options |
| 1 | 1 | 1 | 3µs | 3µs | Hailo::Storage::__ANON__[lib/Hailo/Storage.pm:238] |
| 0 | 0 | 0 | 0s | 0s | Hailo::Storage::_engage_initialized_check_and_set_order |
| 0 | 0 | 0 | 0s | 0s | Hailo::Storage::_engage_initialized_check_and_set_tokenizer |
| 0 | 0 | 0 | 0s | 0s | Hailo::Storage::totals |
| Line | State ments |
Time on line |
Calls | Time in subs |
Code |
|---|---|---|---|---|---|
| 1 | package Hailo::Storage; | ||||
| 2 | |||||
| 3 | 4 | 74µs | 3 | 301µs | use 5.010; # spent 141µs making 1 call to Hailo::Storage::BEGIN@3
# spent 86µs making 1 call to Hailo::Storage::BEGIN@3.12
# spent 73µs making 1 call to feature::import |
| 4 | 2 | 31µs | 2 | 1.19ms | # spent 599µs (13+586) within Hailo::Storage::BEGIN@4 which was called:
# once (13µs+586µs) by Mouse::Util::_try_load_one_class at line 4 # spent 599µs making 1 call to Hailo::Storage::BEGIN@4
# spent 586µs making 1 call to Any::Moose::import |
| 5 | 2 | 29µs | 2 | 667µs | # spent 339µs (11+328) within Hailo::Storage::BEGIN@5 which was called:
# once (11µs+328µs) by Mouse::Util::_try_load_one_class at line 5 # spent 339µs making 1 call to Hailo::Storage::BEGIN@5
# spent 328µs making 1 call to Any::Moose::import |
| 6 | 2 | 159µs | 2 | 13.4ms | # spent 13.4ms (11.1+2.27) within Hailo::Storage::BEGIN@6 which was called:
# once (11.1ms+2.27ms) by Mouse::Util::_try_load_one_class at line 6 # spent 13.4ms making 1 call to Hailo::Storage::BEGIN@6
# spent 53µs making 1 call to Exporter::import |
| 7 | 2 | 1.07ms | 1 | 875µs | # spent 875µs (706+169) within Hailo::Storage::BEGIN@7 which was called:
# once (706µs+169µs) by Mouse::Util::_try_load_one_class at line 7 # spent 875µs making 1 call to Hailo::Storage::BEGIN@7 |
| 8 | |||||
| 9 | 1 | 6µs | 1 | 436µs | has dbd => ( # spent 436µs making 1 call to Mouse::has |
| 10 | isa => 'Str', | ||||
| 11 | is => 'ro', | ||||
| 12 | lazy_build => 1, | ||||
| 13 | documentation => "The DBD::* driver we're using", | ||||
| 14 | ); | ||||
| 15 | |||||
| 16 | 1 | 4µs | 1 | 340µs | has dbd_options => ( # spent 340µs making 1 call to Mouse::has |
| 17 | isa => 'HashRef', | ||||
| 18 | is => 'ro', | ||||
| 19 | lazy_build => 1, | ||||
| 20 | documentation => 'Options passed as the last argument to DBI->connect()', | ||||
| 21 | ); | ||||
| 22 | |||||
| 23 | # spent 5µs within Hailo::Storage::_build_dbd_options which was called:
# once (5µs+0s) by Mouse::super at line 93 of Mouse.pm | ||||
| 24 | 2 | 7µs | my ($self) = @_; | ||
| 25 | return { | ||||
| 26 | RaiseError => 1 | ||||
| 27 | }; | ||||
| 28 | } | ||||
| 29 | |||||
| 30 | 1 | 4µs | 1 | 585µs | has dbh => ( # spent 585µs making 1 call to Mouse::has |
| 31 | isa => 'DBI::db', | ||||
| 32 | is => 'ro', | ||||
| 33 | lazy_build => 1, | ||||
| 34 | documentation => 'Our DBD object', | ||||
| 35 | ); | ||||
| 36 | |||||
| 37 | # spent 46.9ms (22µs+46.9) within Hailo::Storage::_build_dbh which was called:
# once (22µs+46.9ms) by Hailo::Storage::dbh at line 62 of lib/Hailo/Storage/SQLite.pm | ||||
| 38 | 3 | 27µs | my ($self) = @_; | ||
| 39 | 1 | 14µs | 4 | 375µs | my $dbd_options = $self->dbi_options; # spent 195µs making 1 call to Hailo::Storage::dbi_options
# spent 176µs making 1 call to Hailo::Storage::SQLite::_build_dbi_options
# spent 2µs making 1 call to Mouse::Meta::TypeConstraint::_compiled_type_constraint
# spent 1µs making 1 call to Mouse::Meta::Attribute::builder |
| 40 | |||||
| 41 | 2 | 46.7ms | return DBI->connect($self->dbi_options); # spent 46.7ms making 1 call to DBI::connect
# spent 2µs making 1 call to Hailo::Storage::dbi_options | ||
| 42 | }; | ||||
| 43 | |||||
| 44 | 1 | 4µs | 1 | 400µs | has dbi_options => ( # spent 400µs making 1 call to Mouse::has |
| 45 | isa => 'ArrayRef', | ||||
| 46 | is => 'ro', | ||||
| 47 | auto_deref => 1, | ||||
| 48 | lazy_build => 1, | ||||
| 49 | documentation => 'Options passed to DBI->connect()', | ||||
| 50 | ); | ||||
| 51 | |||||
| 52 | # spent 114µs (28+86) within Hailo::Storage::_build_dbi_options which was called:
# once (28µs+86µs) by Hailo::Storage::SQLite::__ANON__[lib/Hailo/Storage/SQLite.pm:36] at line 32 of lib/Hailo/Storage/SQLite.pm | ||||
| 53 | 6 | 42µs | my ($self) = @_; | ||
| 54 | 1 | 12µs | 4 | 32µs | my $dbd = $self->dbd; # spent 26µs making 1 call to Hailo::Storage::dbd
# spent 3µs making 1 call to Hailo::Storage::SQLite::_build_dbd
# spent 1µs making 1 call to Mouse::Meta::Attribute::builder
# spent 1µs making 1 call to Mouse::Meta::TypeConstraint::_compiled_type_constraint |
| 55 | 1 | 15µs | 4 | 100µs | my $dbd_options = $self->dbd_options; # spent 59µs making 1 call to Hailo::Storage::dbd_options
# spent 38µs making 1 call to Hailo::Storage::SQLite::_build_dbd_options
# spent 2µs making 1 call to Mouse::Meta::TypeConstraint::_compiled_type_constraint
# spent 2µs making 1 call to Mouse::Meta::Attribute::builder |
| 56 | 1 | 2µs | my $db = $self->brain // ''; # spent 2µs making 1 call to Hailo::Storage::SQLite::brain | ||
| 57 | |||||
| 58 | my @options = ( | ||||
| 59 | "dbi:$dbd:dbname=$db", | ||||
| 60 | '', | ||||
| 61 | '', | ||||
| 62 | $dbd_options, | ||||
| 63 | ); | ||||
| 64 | |||||
| 65 | return \@options; | ||||
| 66 | } | ||||
| 67 | |||||
| 68 | 1 | 4µs | 1 | 262µs | has _engaged => ( # spent 262µs making 1 call to Mouse::has |
| 69 | isa => 'Bool', | ||||
| 70 | is => 'rw', | ||||
| 71 | default => 0, | ||||
| 72 | documentation => 'Have we done setup work to get this database going?', | ||||
| 73 | ); | ||||
| 74 | |||||
| 75 | 1 | 4µs | 1 | 333µs | has sth => ( # spent 333µs making 1 call to Mouse::has |
| 76 | isa => 'HashRef', | ||||
| 77 | is => 'ro', | ||||
| 78 | lazy_build => 1, | ||||
| 79 | documentation => 'A HashRef of prepared DBI statement handles', | ||||
| 80 | ); | ||||
| 81 | |||||
| 82 | # spent 1.94ms (19µs+1.92) within Hailo::Storage::_build_sth which was called:
# once (19µs+1.92ms) by Hailo::Storage::sth at line 116 | ||||
| 83 | 2 | 22µs | my ($self) = @_; | ||
| 84 | 4 | 1.92ms | return Hailo::Storage::Schema->sth($self->dbd, $self->dbh, $self->order); # spent 1.91ms making 1 call to Hailo::Storage::Schema::sth
# spent 2µs making 1 call to Hailo::Storage::dbh
# spent 1µs making 1 call to Hailo::Storage::dbd
# spent 1µs making 1 call to Hailo::Storage::SQLite::order | ||
| 85 | } | ||||
| 86 | |||||
| 87 | 1 | 3µs | 1 | 264µs | has _boundary_token_id => ( # spent 264µs making 1 call to Mouse::has |
| 88 | isa => 'Int', | ||||
| 89 | is => 'rw', | ||||
| 90 | ); | ||||
| 91 | |||||
| 92 | # bootstrap the database | ||||
| 93 | # spent 5.18ms (115µs+5.06) within Hailo::Storage::_engage which was called:
# once (115µs+5.06ms) by Hailo::Storage::SQLite::_engage at line 332 of Mouse/Meta/Class.pm | ||||
| 94 | 14 | 238µs | my ($self) = @_; | ||
| 95 | |||||
| 96 | 1 | 316µs | if ($self->initialized) { # spent 316µs making 1 call to Hailo::Storage::SQLite::initialized | ||
| 97 | # Check the order we've been given and retrieve it from the | ||||
| 98 | # database if there's nothing odd going on. | ||||
| 99 | $self->_engage_initialized_check_and_set_order; | ||||
| 100 | |||||
| 101 | # Likewise for the Tokenizer | ||||
| 102 | $self->_engage_initialized_check_and_set_tokenizer; | ||||
| 103 | |||||
| 104 | $self->sth->{token_id}->execute(0, ''); | ||||
| 105 | my $id = $self->sth->{token_id}->fetchrow_array; | ||||
| 106 | $self->_boundary_token_id($id); | ||||
| 107 | } | ||||
| 108 | else { | ||||
| 109 | 4 | 2.62ms | Hailo::Storage::Schema->deploy($self->dbd, $self->dbh, $self->order); # spent 2.61ms making 1 call to Hailo::Storage::Schema::deploy
# spent 2µs making 1 call to Hailo::Storage::dbd
# spent 1µs making 1 call to Hailo::Storage::SQLite::order
# spent 1µs making 1 call to Hailo::Storage::dbh | ||
| 110 | |||||
| 111 | # Set metadata in the database for use by subsequent | ||||
| 112 | # invocations | ||||
| 113 | { | ||||
| 114 | # Don't change order again | ||||
| 115 | 1 | 2µs | my $order = $self->order; # spent 2µs making 1 call to Hailo::Storage::SQLite::order | ||
| 116 | 1 | 76µs | 5 | 3.96ms | $self->sth->{set_info}->execute('markov_order', $order); # spent 1.97ms making 1 call to Hailo::Storage::sth
# spent 1.94ms making 1 call to Hailo::Storage::_build_sth
# spent 53µs making 1 call to DBI::st::execute
# spent 2µs making 1 call to Mouse::Meta::Attribute::builder
# spent 2µs making 1 call to Mouse::Meta::TypeConstraint::_compiled_type_constraint |
| 117 | |||||
| 118 | # Warn if the tokenizer changes | ||||
| 119 | 1 | 2µs | my $tokenizer = $self->tokenizer_class; # spent 2µs making 1 call to Hailo::Storage::SQLite::tokenizer_class | ||
| 120 | 2 | 32µs | $self->sth->{set_info}->execute('tokenizer_class', $tokenizer); # spent 31µs making 1 call to DBI::st::execute
# spent 1µs making 1 call to Hailo::Storage::sth | ||
| 121 | } | ||||
| 122 | |||||
| 123 | 2 | 36µs | $self->sth->{add_token}->execute(0, ''); # spent 34µs making 1 call to DBI::st::execute
# spent 2µs making 1 call to Hailo::Storage::sth | ||
| 124 | 2 | 16µs | $self->sth->{last_token_rowid}->execute(); # spent 15µs making 1 call to DBI::st::execute
# spent 2µs making 1 call to Hailo::Storage::sth | ||
| 125 | 2 | 11µs | my $id = $self->sth->{last_token_rowid}->fetchrow_array(); # spent 10µs making 1 call to DBI::st::fetchrow_array
# spent 1µs making 1 call to Hailo::Storage::sth | ||
| 126 | 2 | 13µs | $self->_boundary_token_id($id); # spent 11µs making 1 call to Hailo::Storage::_boundary_token_id
# spent 2µs making 1 call to Mouse::Meta::TypeConstraint::_compiled_type_constraint | ||
| 127 | } | ||||
| 128 | |||||
| 129 | 1 | 2µs | $self->_engaged(1); # spent 2µs making 1 call to Hailo::Storage::_engaged | ||
| 130 | |||||
| 131 | return; | ||||
| 132 | } | ||||
| 133 | |||||
| 134 | sub _engage_initialized_check_and_set_order { | ||||
| 135 | my ($self) = @_; | ||||
| 136 | |||||
| 137 | my $sth = $self->dbh->prepare(qq[SELECT text FROM info WHERE attribute = ?;]); | ||||
| 138 | $sth->execute('markov_order'); | ||||
| 139 | my $db_order = $sth->fetchrow_array(); | ||||
| 140 | |||||
| 141 | my $my_order = $self->order; | ||||
| 142 | if ($my_order != $db_order) { | ||||
| 143 | if ($self->hailo->{has_custom_order}->()) { | ||||
| 144 | die <<"DIE"; | ||||
| 145 | You've manually supplied an order of `$my_order' to Hailo but you're | ||||
| 146 | loading a brain that has the order `$db_order'. | ||||
| 147 | |||||
| 148 | Hailo will automatically load the order from existing brains, however | ||||
| 149 | you've constructed Hailo and manually specified an order not | ||||
| 150 | equivalent to the existing order of the database. | ||||
| 151 | |||||
| 152 | Either supply the correct order or omit the order attribute | ||||
| 153 | altogether. We could continue but I'd rather die since you're probably | ||||
| 154 | expecting something I can't deliver. | ||||
| 155 | DIE | ||||
| 156 | } | ||||
| 157 | |||||
| 158 | $self->order($db_order); | ||||
| 159 | $self->hailo->{set_order}->($db_order); | ||||
| 160 | } | ||||
| 161 | |||||
| 162 | return; | ||||
| 163 | } | ||||
| 164 | |||||
| 165 | sub _engage_initialized_check_and_set_tokenizer { | ||||
| 166 | my ($self) = @_; | ||||
| 167 | |||||
| 168 | my $sth = $self->dbh->prepare(qq[SELECT text FROM info WHERE attribute = ?;]); | ||||
| 169 | $sth->execute('tokenizer_class'); | ||||
| 170 | my $db_tokenizer_class = $sth->fetchrow_array; | ||||
| 171 | my $my_tokenizer_class = $self->tokenizer_class; | ||||
| 172 | |||||
| 173 | # defined() because we can't count on old brains having this | ||||
| 174 | if (defined $db_tokenizer_class | ||||
| 175 | and $my_tokenizer_class ne $db_tokenizer_class) { | ||||
| 176 | if ($self->hailo->{has_custom_tokenizer_class}->()) { | ||||
| 177 | die <<"DIE"; | ||||
| 178 | You've manually supplied a tokenizer class `$my_tokenizer_class' to | ||||
| 179 | Hailo, but you're loading a brain that has the tokenizer class | ||||
| 180 | `$db_tokenizer_class'. | ||||
| 181 | |||||
| 182 | Hailo will automatically load the tokenizer class from existing | ||||
| 183 | brains, however you've constructed Hailo and manually specified an | ||||
| 184 | tokenizer class not equivalent to the existing tokenizer class of the | ||||
| 185 | database. | ||||
| 186 | |||||
| 187 | Either supply the correct tokenizer class or omit the order attribute | ||||
| 188 | altogether. We could continue but I'd rather die since you're probably | ||||
| 189 | expecting something I can't deliver. | ||||
| 190 | DIE | ||||
| 191 | } | ||||
| 192 | |||||
| 193 | $self->tokenizer_class($db_tokenizer_class); | ||||
| 194 | $self->hailo->{set_tokenizer_class}->($db_tokenizer_class); | ||||
| 195 | } | ||||
| 196 | |||||
| 197 | return; | ||||
| 198 | } | ||||
| 199 | |||||
| 200 | # spent 5.61ms (28µs+5.58) within Hailo::Storage::start_training which was called:
# once (28µs+5.58ms) by Hailo::Storage::SQLite::start_training at line 332 of Mouse/Meta/Class.pm | ||||
| 201 | 4 | 28µs | my ($self) = @_; | ||
| 202 | 2 | 5.51ms | $self->_engage() unless $self->_engaged; # spent 5.51ms making 1 call to Hailo::Storage::SQLite::_engage
# spent 2µs making 1 call to Hailo::Storage::_engaged | ||
| 203 | 1 | 66µs | $self->start_learning(); # spent 66µs making 1 call to Hailo::Storage::start_learning | ||
| 204 | return; | ||||
| 205 | } | ||||
| 206 | |||||
| 207 | # spent 4.07ms (19µs+4.05) within Hailo::Storage::stop_training which was called:
# once (19µs+4.05ms) by Hailo::Storage::SQLite::stop_training at line 349 of Mouse/Meta/Class.pm | ||||
| 208 | 3 | 17µs | my ($self) = @_; | ||
| 209 | 1 | 4.05ms | $self->stop_learning(); # spent 4.05ms making 1 call to Hailo::Storage::stop_learning | ||
| 210 | return; | ||||
| 211 | } | ||||
| 212 | |||||
| 213 | # spent 66µs (18+48) within Hailo::Storage::start_learning which was called:
# once (18µs+48µs) by Hailo::Storage::start_training at line 203 | ||||
| 214 | 4 | 24µs | my ($self) = @_; | ||
| 215 | 1 | 1µs | $self->_engage() unless $self->_engaged; # spent 1µs making 1 call to Hailo::Storage::_engaged | ||
| 216 | |||||
| 217 | # start a transaction | ||||
| 218 | 1 | 6µs | 3 | 82µs | $self->dbh->begin_work; # spent 45µs making 1 call to DBI::db::begin_work
# spent 36µs making 1 call to DBD::_::db::begin_work
# spent 1µs making 1 call to Hailo::Storage::dbh |
| 219 | return; | ||||
| 220 | } | ||||
| 221 | |||||
| 222 | # spent 4.05ms (42µs+4.01) within Hailo::Storage::stop_learning which was called:
# once (42µs+4.01ms) by Hailo::Storage::stop_training at line 209 | ||||
| 223 | 3 | 4.05ms | my ($self) = @_; | ||
| 224 | # finish a transaction | ||||
| 225 | 2 | 4.01ms | $self->dbh->commit; # spent 4.00ms making 1 call to DBI::db::commit
# spent 2µs making 1 call to Hailo::Storage::dbh | ||
| 226 | return; | ||||
| 227 | } | ||||
| 228 | |||||
| 229 | # See if SELECT count(*) FROM info; fails. If not we assume that we | ||||
| 230 | # have an up and running database. | ||||
| 231 | # spent 259µs (120+140) within Hailo::Storage::initialized which was called:
# once (120µs+140µs) by Mouse::super at line 93 of Mouse.pm | ||||
| 232 | 8 | 119µs | my ($self) = @_; | ||
| 233 | 1 | 2µs | my $dbh = $self->dbh; # spent 2µs making 1 call to Hailo::Storage::dbh | ||
| 234 | |||||
| 235 | my ($err, $warn, $res); | ||||
| 236 | eval { | ||||
| 237 | # SQLite will warn 'no such table info' | ||||
| 238 | 1 | 6µs | # spent 3µs within Hailo::Storage::__ANON__[lib/Hailo/Storage.pm:238] which was called:
# once (3µs+0s) by DBI::db::do at line 242 | ||
| 239 | |||||
| 240 | # If it doesn't warn trust that it dies here | ||||
| 241 | local ($@, $!); | ||||
| 242 | 1 | 26µs | 3 | 251µs | $res = $dbh->do("SELECT count(*) FROM info;"); # spent 138µs making 1 call to DBI::db::do
# spent 109µs making 1 call to DBD::SQLite::db::do
# spent 3µs making 1 call to Hailo::Storage::__ANON__[lib/Hailo/Storage.pm:238] |
| 243 | }; | ||||
| 244 | |||||
| 245 | return (not $err and not $warn and defined $res); | ||||
| 246 | } | ||||
| 247 | |||||
| 248 | # return some statistics | ||||
| 249 | sub totals { | ||||
| 250 | my ($self) = @_; | ||||
| 251 | $self->_engage() unless $self->_engaged; | ||||
| 252 | |||||
| 253 | $self->sth->{token_total}->execute(); | ||||
| 254 | my $token = $self->sth->{token_total}->fetchrow_array - 1; | ||||
| 255 | $self->sth->{expr_total}->execute(); | ||||
| 256 | my $expr = $self->sth->{expr_total}->fetchrow_array // 0; | ||||
| 257 | $self->sth->{prev_total}->execute(); | ||||
| 258 | my $prev = $self->sth->{prev_total}->fetchrow_array // 0; | ||||
| 259 | $self->sth->{next_total}->execute(); | ||||
| 260 | my $next = $self->sth->{next_total}->fetchrow_array // 0; | ||||
| 261 | |||||
| 262 | return $token, $expr, $prev, $next; | ||||
| 263 | } | ||||
| 264 | |||||
| 265 | 1 | 21µs | 2 | 138µs | __PACKAGE__->meta->make_immutable; # spent 124µs making 1 call to Mouse::Meta::Class::make_immutable
# spent 15µs making 1 call to Hailo::Storage::meta |
| 266 | |||||
| 267 | =encoding utf8 | ||||
| 268 | |||||
| 269 | =head1 NAME | ||||
| 270 | |||||
| 271 | Hailo::Storage - A base class for L<Hailo> L<storage|Hailo::Role::Storage> backends | ||||
| 272 | |||||
| 273 | =head1 METHODS | ||||
| 274 | |||||
| 275 | The following methods must to be implemented by subclasses: | ||||
| 276 | |||||
| 277 | =head2 C<_build_dbd> | ||||
| 278 | |||||
| 279 | Should return the name of the database driver (e.g. 'SQLite') which will be | ||||
| 280 | passed to L<DBI|DBI>. | ||||
| 281 | |||||
| 282 | =head2 C<_build_dbd_options> | ||||
| 283 | |||||
| 284 | Subclasses can override this method to add options of their own. E.g: | ||||
| 285 | |||||
| 286 | override _build_dbd_options => sub { | ||||
| 287 | return { | ||||
| 288 | %{ super() }, | ||||
| 289 | sqlite_unicode => 1, | ||||
| 290 | }; | ||||
| 291 | }; | ||||
| 292 | |||||
| 293 | =head2 C<initialized> | ||||
| 294 | |||||
| 295 | Should return a true value if the database has already been created. | ||||
| 296 | |||||
| 297 | =head1 Comparison of backends | ||||
| 298 | |||||
| 299 | This benchmark shows how the backends compare when training on the | ||||
| 300 | small testsuite dataset as reported by the F<utils/hailo-benchmark> | ||||
| 301 | utility (found in the distribution): | ||||
| 302 | |||||
| 303 | Rate DBD::Pg DBD::mysql DBD::SQLite/file DBD::SQLite/memory | ||||
| 304 | DBD::Pg 2.22/s -- -33% -49% -56% | ||||
| 305 | DBD::mysql 3.33/s 50% -- -23% -33% | ||||
| 306 | DBD::SQLite/file 4.35/s 96% 30% -- -13% | ||||
| 307 | DBD::SQLite/memory 5.00/s 125% 50% 15% -- | ||||
| 308 | |||||
| 309 | Under real-world workloads SQLite is much faster than these results | ||||
| 310 | indicate since the time it takes to train/reply is relative to the | ||||
| 311 | existing database size. Here's how long it took to train on a 214,710 | ||||
| 312 | line IRC log on a Linode 1080 with Hailo 0.18: | ||||
| 313 | |||||
| 314 | =over | ||||
| 315 | |||||
| 316 | =item * SQLite | ||||
| 317 | |||||
| 318 | real 8m38.285s | ||||
| 319 | user 8m30.831s | ||||
| 320 | sys 0m1.175s | ||||
| 321 | |||||
| 322 | =item * MySQL | ||||
| 323 | |||||
| 324 | real 48m30.334s | ||||
| 325 | user 8m25.414s | ||||
| 326 | sys 4m38.175s | ||||
| 327 | |||||
| 328 | =item * PostgreSQL | ||||
| 329 | |||||
| 330 | real 216m38.906s | ||||
| 331 | user 11m13.474s | ||||
| 332 | sys 4m35.509s | ||||
| 333 | |||||
| 334 | =back | ||||
| 335 | |||||
| 336 | In the case of PostgreSQL it's actually much faster to first train | ||||
| 337 | with SQLite, dump that database and then import it with L<psql(1)>, | ||||
| 338 | see L<failo's README|http://github.com/hinrik/failo> for how to do | ||||
| 339 | that. | ||||
| 340 | |||||
| 341 | However when replying with an existing database (using | ||||
| 342 | F<utils/hailo-benchmark-replies>) yields different results. SQLite can | ||||
| 343 | reply really quickly without being warmed up (which is the typical | ||||
| 344 | usecase for chatbots) but once PostgreSQL and MySQL are warmed up they | ||||
| 345 | start replying faster: | ||||
| 346 | |||||
| 347 | Here's a comparison of doing 10 replies: | ||||
| 348 | |||||
| 349 | Rate PostgreSQL MySQL SQLite-file SQLite-file-28MB SQLite-memory | ||||
| 350 | PostgreSQL 71.4/s -- -14% -14% -29% -50% | ||||
| 351 | MySQL 83.3/s 17% -- 0% -17% -42% | ||||
| 352 | SQLite-file 83.3/s 17% 0% -- -17% -42% | ||||
| 353 | SQLite-file-28MB 100.0/s 40% 20% 20% -- -30% | ||||
| 354 | SQLite-memory 143/s 100% 71% 71% 43% -- | ||||
| 355 | |||||
| 356 | In this test MySQL uses around 28MB of memory (using Debian's | ||||
| 357 | F<my-small.cnf>) and PostgreSQL around 34MB. Plain SQLite uses 2MB of | ||||
| 358 | cache but it's also tested with 28MB of cache as well as with the | ||||
| 359 | entire database in memory. | ||||
| 360 | |||||
| 361 | But doing 10,000 replies is very different: | ||||
| 362 | |||||
| 363 | Rate SQLite-file PostgreSQL SQLite-file-28MB MySQL SQLite-memory | ||||
| 364 | SQLite-file 85.1/s -- -7% -18% -27% -38% | ||||
| 365 | PostgreSQL 91.4/s 7% -- -12% -21% -33% | ||||
| 366 | SQLite-file-28MB 103/s 21% 13% -- -11% -25% | ||||
| 367 | MySQL 116/s 37% 27% 13% -- -15% | ||||
| 368 | SQLite-memory 137/s 61% 50% 33% 18% -- | ||||
| 369 | |||||
| 370 | Once MySQL gets more memory (using Debian's F<my-large.cnf>) and a | ||||
| 371 | chance to warm it starts yielding better results (I couldn't find out | ||||
| 372 | how to make PostgreSQL take as much memory as it wanted): | ||||
| 373 | |||||
| 374 | Rate MySQL SQLite-memory | ||||
| 375 | MySQL 121/s -- -12% | ||||
| 376 | SQLite-memory 138/s 14% -- | ||||
| 377 | |||||
| 378 | =head1 AUTHOR | ||||
| 379 | |||||
| 380 | E<AElig>var ArnfjE<ouml>rE<eth> Bjarmason <avar@cpan.org> | ||||
| 381 | |||||
| 382 | Hinrik E<Ouml>rn SigurE<eth>sson, hinrik.sig@gmail.com | ||||
| 383 | |||||
| 384 | =head1 LICENSE AND COPYRIGHT | ||||
| 385 | |||||
| 386 | Copyright 2010 E<AElig>var ArnfjE<ouml>rE<eth> Bjarmason and | ||||
| 387 | Hinrik E<Ouml>rn SigurE<eth>sson | ||||
| 388 | |||||
| 389 | This program is free software, you can redistribute it and/or modify | ||||
| 390 | it under the same terms as Perl itself. | ||||
| 391 | |||||
| 392 | =cut | ||||
# spent 200ms (200+2µs) within Hailo::Storage::_boundary_token_id which was called 131075 times, avg 2µs/call:
# 131074 times (200ms+0s) by Hailo::Engine::Default::learn at line 133 of lib/Hailo/Engine/Default.pm, avg 2µs/call
# once (9µs+2µs) by Hailo::Storage::_engage at line 126 | |||||
# spent 8µs within Hailo::Storage::_engaged which was called 4 times, avg 2µs/call:
# once (2µs+0s) by Hailo::Storage::_engage at line 129
# once (2µs+0s) by Hailo::Storage::start_training at line 202
# once (2µs+0s) by Hailo::Storage::SQLite::save at line 120 of lib/Hailo/Storage/SQLite.pm
# once (1µs+0s) by Hailo::Storage::start_learning at line 215 | |||||
sub Hailo::Storage::dbd; # xsub | |||||
# spent 59µs (18+41) within Hailo::Storage::dbd_options which was called:
# once (18µs+41µs) by Hailo::Storage::_build_dbi_options at line 55 | |||||
# spent 180ms (133+46.9) within Hailo::Storage::dbh which was called 98607 times, avg 2µs/call:
# 77849 times (105ms+0s) by Hailo::Engine::Default::_add_expr at line 184 of lib/Hailo/Engine/Default.pm, avg 1µs/call
# 20749 times (27.9ms+0s) by Hailo::Engine::Default::_add_token at line 225 of lib/Hailo/Engine/Default.pm, avg 1µs/call
# 2 times (3µs+0s) by Hailo::Storage::SQLite::_set_pragmas at line 109 of lib/Hailo/Storage/SQLite.pm, avg 2µs/call
# once (30µs+46.9ms) by Hailo::Storage::SQLite::__ANON__[lib/Hailo/Storage/SQLite.pm:66] at line 62 of lib/Hailo/Storage/SQLite.pm
# once (4µs+0s) by Hailo::Storage::SQLite::__ANON__[lib/Hailo/Storage/SQLite.pm:73] at line 69 of lib/Hailo/Storage/SQLite.pm
# once (2µs+0s) by Hailo::Storage::stop_learning at line 225
# once (2µs+0s) by Hailo::Storage::initialized at line 233
# once (2µs+0s) by Hailo::Storage::_build_sth at line 84
# once (1µs+0s) by Hailo::Storage::_engage at line 109
# once (1µs+0s) by Hailo::Storage::start_learning at line 218 | |||||
sub Hailo::Storage::dbi_options; # xsub | |||||
# spent 1.97ms (34µs+1.94) within Hailo::Storage::sth which was called 6 times, avg 329µs/call:
# once (26µs+1.94ms) by Hailo::Storage::_engage at line 116
# once (2µs+0s) by Hailo::Engine::Default::BUILD at line 29 of lib/Hailo/Engine/Default.pm
# once (2µs+0s) by Hailo::Storage::_engage at line 124
# once (2µs+0s) by Hailo::Storage::_engage at line 123
# once (1µs+0s) by Hailo::Storage::_engage at line 120
# once (1µs+0s) by Hailo::Storage::_engage at line 125 |