##========================================================================
## POD DOCUMENTATION, auto-generated by podextract.perl

##========================================================================
## NAME
=pod

=head1 NAME

DiaColloDB::Relation::TDF - collocation db, profiling relation: (term x document) raw-frequency matrix

=cut

##========================================================================
## SYNOPSIS
=pod

=head1 SYNOPSIS

 ##========================================================================
 ## PRELIMINARIES
 
 use DiaColloDB::Relation::TDF;
 
 ##========================================================================
 ## Constructors etc.
 
 $rel = CLASS_OR_OBJECT->new(%args);
 
 ##========================================================================
 ## TDF API: Utils
 
 $vtype = $rel->vtype();
 $itype = $rel->itype();
 $packas = $rel->vpack();
 $packas = $rel->ipack();
 
 ##========================================================================
 ## Persistent API: disk usage
 
 @files = $rel->diskFiles();
 
 ##========================================================================
 ## Persistent API: header
 
 @keys = $rel->headerKeys();
 $hdr = $rel->headerData();
 
 ##========================================================================
 ## Relation API: open/close
 
 $rel_or_undef = $rel->open($base);
 $rel_or_undef = $rel->close();
 $bool = $rel->opened();
 
 ##========================================================================
 ## Relation API: create
 
 $rel = $CLASS_OR_OBJECT->create($coldb,$tokdat_file,%opts);
 
 ##========================================================================
 ## Relation API: union
 
 $rel = CLASS_OR_OBJECT->union($coldb, \@dbargs, %opts);
 
 ##========================================================================
 ## Relation API: dbinfo
 
 \%info = $rel->dbinfo($coldb);
 
 ##========================================================================
 ## Relation API: profile
 
 $mprf = $rel->profile($coldb, %opts);
 
 ##========================================================================
 ## Relation API: comparison (diff)
 
 $mpdiff = $rel->compare($coldb, %opts);
 
 ##========================================================================
 ## Profile: Utils: PDL-based profiling
 
 \@pprfs = $rel->vprofile($coldb, \%opts);
 
 ##========================================================================
 ## Profile: Utils: domain sizes
 
 $NT = $rel->nTerms();
 $ND = $rel->nDocs();
 $NC = $rel->nFiles();
 $NA = $rel->nAttrs();
 $NM = $rel->nMeta();
 
 ##========================================================================
 ## Profile: Utils: attribute positioning
 
 \%tpos = $rel->tpos();
 \%mpos = $rel->mpos();
 
 ##========================================================================
 ## Profile: Utils: query parsing & evaluation
 
 $idPdl    = $rel->idpdl($idPdl);
 $tupleIds = $rel->tupleIds($attrType, $attrName, $valIdsPdl);
 $ti       = $rel->termIds($tattrName, $valIdsPDL);
 $ci       = $rel->catIds($mattrName, $valIdsPDL);
 
 $bool          = $rel->hasMeta($attr);
 $enum_or_undef = $rel->metaEnum($mattr);
 
 $cats          = $rel->catSubset($terms);
 
 \%groupby      = $rel->groupby($coldb, $groupby_request, %opts);
 
 ##========================================================================
 ## Relation API: default: query info
 
 \%qinfo = $rel->qinfo($coldb, %opts);

=cut

##========================================================================
## DESCRIPTION
=pod

=head1 DESCRIPTION

DiaColloDB::Relation::TDF
is a L<DiaColloDB::Relation|DiaColloDB::Relation> subclass for
document-level co-occurrence frequencies using
L<PDL|PDL> to efficiently store and query
a sparse underlying (term x document) frequency matrix
via the L<PDL::CCS|PDL::CCS> package.

Supports Boolean expressions over both term- and document-level
conditions (the latter via DDC C<#has[ATTRIBUTE,VALUE]> or
C<#has[ATTRIBUTE,/REGEX/]> syntax) as well as grouping via literal
indexed term- and/or document-level attributes.

An earlier version of this module was implemented
as C<DiaColloDB::Relation::Vsem> ("vector-space distributional semantic index").

=cut

##----------------------------------------------------------------
## DESCRIPTION: DiaColloDB::Relation::TDF: Globals & Constants
=pod

=head2 Globals & Constants

=over 4

=item Variable: @ISA

DiaColloDB::Relation::TDF inherits from
L<DiaColloDB::Relation|DiaColloDB::Relation>.

=back

=cut

##----------------------------------------------------------------
## DESCRIPTION: DiaColloDB::Relation::TDF: Constructors etc.
=pod

=head2 Constructors etc.

=over 4

=item new

 $rel = CLASS_OR_OBJECT->new(%args);

%args, object structure:

 ##-- user options
 base   => $basename,   ##-- relation basename
 flags  => $flags,      ##-- i/o flags (default: 'r')
 mgood  => $regex,      ##-- positive filter regex for metadata attributes
 mbad   => $regex,      ##-- negative filter regex for metadata attributes
 submax => $submax,     ##-- choke on requested tdm cross-subsets if dense subset size ($NT_sub * $ND_sub) > $submax; default=2**29 (512M)
 mquery => \%mquery,    ##-- qinfo templates for meta-fields (default: textClass hack for genre): ($mattr=>$TEMPLATE, ...)
 ##
 ##-- logging options
 logvprofile => $level, ##-- log-level for vprofile() (default=undef:none)
 logio => $level,       ##-- log-level for low-level I/O operations (default=undef:none)
 ##
 ##-- modelling options (formerly via DocClassify)
 minFreq    => $fmin,   ##-- minimum total term-frequency for model inclusion (default=undef:use $coldb->{tfmin})
 minDocFreq => $dfmin,  ##-- minimim "doc-frequency" (#/docs per term) for model inclusion (default=4)
 minDocSize => $dnmin,  ##-- minimum doc size (#/tokens per doc) for model inclusion (default=4; formerly $coldb->{vbnmin})
 maxDocSize => $dnmax,  ##-- maximum doc size (#/tokens per doc) for model inclusion (default=inf; formerly $coldb->{vbnmax})
 vtype      => $vtype,  ##-- PDL::Type for storing compiled values (default=float; auto-promoted if required)
 itype      => $itype,  ##-- PDL::Type for storing compiled integers (default=long)
 ##
 ##-- guts: aux: info
 N => $tdm0Total,       ##-- total number of (doc,term) frequencies counted
 dbreak => $dbreak,     ##-- inherited from $coldb on create()
 ##
 ##-- guts: aux: term-tuples ($NA:number of term-attributes, $NT:number of term-tuples)
 attrs  => \@attrs,       ##-- known term attributes
 tvals  => $tvals,        ##-- pdl($NA,$NT) : [$apos,$ti] => $avali_at_term_ti
 tsorti => $tsorti,       ##-- pdl($NT,$NA) : [,($apos)]  => $tvals->slice("($apos),")->qsorti
 tpos   => \%a2pos,       ##-- term-attribute positions: $apos=$a2pos{$aname}
 ##
 ##-- guts: aux: metadata ($NM:number of metas-attributes, $NC:number of cats (source files))
 meta => \@mattrs         ##-- known metadata attributes
 meta_e_${ATTR} => $enum, ##-- metadata-attribute enum
 mvals => $mvals,         ##-- pdl($NM,$NC) : [$mpos,$ci] => $mvali_at_ci
 msorti => $msorti,       ##-- pdl($NC,$NM) : [,($mpos)]  => $mvals->slice("($mpos),")->qsorti
 mpos  => \%m2pos,        ##-- meta-attribute positions: $mpos=$m2pos{$mattr}
 ##
 ##-- guts: model (formerly via DocClassify dcmap=>$dcmap)
 tdm => $tdm,             ##-- term-doc matrix : PDL::CCS::Nd ($NT,$ND): [$ti,$di] -> f($ti,$di)
 tym => $tym,             ##-- term-year matrix: PDL::CCS::Nd ($NT,$NY): [$ti,$yi] -> f($ti,$yi)
 cf  => $cf_pdl,          ##-- cat-freq pdl:     dense:       ($NC)    : [$ci]     -> f($ci)
 c2date => $c2date,       ##-- cat-dates   : dense ($NC)   : [$ci]   -> $date
 c2d    => $c2d,          ##-- cat->doc map: dense (2,$NC) : [*,$ci] -> [$di_off,$di_len]
 d2c    => $d2c,          ##-- doc->cat map: dense ($ND)   : [$di]   -> $ci
 #...

=back

=cut

##----------------------------------------------------------------
## DESCRIPTION: DiaColloDB::Relation::TDF: TDF API: Utils
=pod

=head2 TDF API: Utils

=over 4

=item vtype

 $vtype = $rel->vtype();

get L<PDL::Type|PDL::Type> value type for storing compiled values.

=item itype

 $itype = $rel->itype();

get L<PDL::Type|PDL::Type> integer type for storing compiled indices.

=item vpack

 $packas = $rel->vpack();

pack-template for $rel-E<gt>vtype(), e.g. "f*"

=item ipack

 $packas = $rel->ipack();

pack-template for $rel-E<gt>itype(), e.g. "l*"

=back

=cut

##----------------------------------------------------------------
## DESCRIPTION: DiaColloDB::Relation::TDF: Persistent API: disk usage
=pod

=head2 Persistent API: disk usage

=over 4

=item diskFiles

 @files = $rel->diskFiles();

returns disk storage files, used by du() and timestamp()

=back

=cut

##----------------------------------------------------------------
## DESCRIPTION: DiaColloDB::Relation::TDF: Persistent API: header
=pod

=head2 Persistent API: header

=over 4

=item headerKeys

 @keys = $rel->headerKeys();

keys to save as header;
override includes qw(meta attrs vtype itype) and excludes logging and i/o keys.

=item headerData

 $hdr = $rel->headerData();

returns reference to object header data;
override stringifies {itype} and {vtype} keys.

=back

=cut

##----------------------------------------------------------------
## DESCRIPTION: DiaColloDB::Relation::TDF: Relation API: open/close
=pod

=head2 Relation API: open/close

=over 4

=item open

 $rel_or_undef = $rel->open($base);
 $rel_or_undef = $rel->open($base,$flags);
 $rel_or_undef = $rel->open();

Opens underlying index files.

=item close

 $rel_or_undef = $rel->close();

Closes underlying index files.

=item opened

 $bool = $rel->opened();

Returns true iff index is opened.
Really just checks for C<$rel-E<gt>{tdm}>.

=back

=cut

##----------------------------------------------------------------
## DESCRIPTION: DiaColloDB::Relation::TDF: Relation API: create
=pod

=head2 Relation API: create

=over 4

=item create

 $rel = $CLASS_OR_OBJECT->create($coldb,$tokdat_file,%opts);

Populates relation index for $coldb.
Requires:

=over 4

=item *

(temporary, tied) doc-arrays @$coldb{qw(docmeta docoff)}

=item *

temp file "$coldb-E<gt>{dbdir}/vtokens.bin": pack($coldb-E<gt>{pack_w}, @wattrs)

OR

wdmfile=E<gt>$wdmfile option

=back

%opts: clobber %$rel, also:

 docmeta =>\@docmeta, ##-- for union(): override $coldb->{docmeta}
                      ##   $docmeta[$ci] = {id=>$id, nsigs=>$nsigs, file=>$rawfile, date=>$date, label=>$label, meta=>\%meta}
 wdmfile =>$wdmfile,  ##-- for union(): txt ~ "$ai0 $ai1 ... $aiN $doci $f"; default is generated from 'vtokens.bin'
 ivalmax =>$imax,     ##-- for union(): maximum integer value (for auto-promotion)
 reusedir=>$bool,     ##-- for union(): set to true if we're running in a "clean" directory
 logas   =>$logas,    ##-- log label (default: 'create()')

=back

=cut

##----------------------------------------------------------------
## DESCRIPTION: DiaColloDB::Relation::TDF: Relation API: union
=pod

=head2 Relation API: union

=over 4

=item union

 $rel = CLASS_OR_OBJECT->union($coldb, \@dbargs, %opts);

merge multiple tdf indices into new object.
\@dbargs is an ARRAY-ref of L<DiaColloDB|DiaColloDB>
sub-objects ($coldb,...) containing {tdf} relations to be merged.

%opts: clobber %$rel

Current implementation
just creates temp-files F<utdm0.dat> and F<udocmeta.tmp> and then calls create().

=back

=cut

##----------------------------------------------------------------
## DESCRIPTION: DiaColloDB::Relation::TDF: Relation API: dbinfo
=pod

=head2 Relation API: dbinfo

=over 4

=item dbinfo

 \%info = $rel->dbinfo($coldb);

embedded info-hash for $coldb-E<gt>dbinfo()

=back

=cut

##----------------------------------------------------------------
## DESCRIPTION: DiaColloDB::Relation::TDF: Relation API: profile
=pod

=head2 Relation API: profiling

=over 4

=item profile

 $mprf = $rel->profile($coldb, %opts);

Get a relation profile for selected items as a L<DiaColloDB::Profile::Multi|DiaColloDB::Profile::Multi> object.
%opts are as for DiaColloDB::Relation::profile().
Really just a wrapper for
the L<vprofile()|/vprofile> method.

=back

=cut

##----------------------------------------------------------------
## DESCRIPTION: DiaColloDB::Relation::TDF: Relation API: comparison (diff)
=pod

=head2 Relation API: comparison (diff)

=over 4

=item compare

 $mpdiff = $rel->compare($coldb, %opts);

Get a relation comparison profile for selected items as a
L<DiaColloDB::Profile::MultiDiff|DiaColloDB::Profile::MultiDiff>
object.
%opts are as for
L<DiaColloDB::Relation::compare()|DiaColloDB::Relation/compare>,
which this method calls after parsing the
C<groupby> option via
L<$rel-E<gt>groupby($coldb, $opts{groupby}, relax=E<gt>0)|/groupby>.

=back

=cut

##----------------------------------------------------------------
## DESCRIPTION: DiaColloDB::Relation::TDF: Profile: Utils: PDL-based profiling
=pod

=head2 Profile: Utils: PDL-based profiling

=over 4

=item vprofile

 \@pprfs = $rel->vprofile($coldb, \%opts);

Guts for the L<profile()|/profile> method.
User options in %opts are as for
L<DiaColloDB::Relation::profile()|DiaColloDB::Relation/profile>.
Additional keys are populated and used in the course of the computation
(so don't set them):

 vq      => $vq,        ##-- parsed query, DiaColloDB::Relation::TDF::Query object
 groubpy => \%groupby,  ##-- as returned by $rel->groupby($coldb, \%opts)
 dlo     => $dlo,       ##-- as returned by $coldb->parseDateRequest(@opts{qw(date slice fill)},1);
 dhi     => $dhi,       ##-- as returned by $coldb->parseDateRequest(@opts{qw(date slice fill)},1);
 dslo    => $dslo,      ##-- as returned by $coldb->parseDateRequest(@opts{qw(date slice fill)},1);
 dshi    => $dshi,      ##-- as returned by $coldb->parseDateRequest(@opts{qw(date slice fill)},1);

=back

=cut

##----------------------------------------------------------------
## DESCRIPTION: DiaColloDB::Relation::TDF: Profile: Utils: domain sizes
=pod

=head2 Profile: Utils: domain sizes

=over 4

=item nTerms

 $NT = $rel->nTerms();

returns number of indexed terms.

=item nDocs

 $ND = $rel->nDocs();

returns number of indexed documents (breaks).

=item nFiles

 $NC = $rel->nFiles();

returns number of indexed categories (original source files).

=item nAttrs

 $NA = $rel->nAttrs();

returns number of indexed term-attributes.

=item nMeta

 $NM = $rel->nMeta();

returns number of indexed meta-attributes.

=back

=cut

##----------------------------------------------------------------
## DESCRIPTION: DiaColloDB::Relation::TDF: Profile: Utils: attribute positioning
=pod

=head2 Profile: Utils: attribute positioning

=over 4

=item tpos

 \%tpos = $rel->tpos();
 $tpos  = $rel->tpos($tattr);

In the first form, get or build the term-attribute position lookup hash.
In the second form, get the index position along dimension $NA
of the term-attribute named C<$tattr>, or undef if C<$tattr> is not a known
term attribute.

=item mpos

 \%mpos = $rel->mpos();
  $mpos  = $rel->mpos($mattr);

In the first form, get or build the meta-attribute position lookup hash.
In the second form, get the index position along dimension $NM
of the meta-attribute named C<$mattr>, or undef if C<$mattr> is not a known
metadata attribute.

=back

=cut

##----------------------------------------------------------------
## DESCRIPTION: DiaColloDB::Relation::TDF: Profile: Utils: query parsing & evaluation
=pod

=head2 Profile: Utils: query parsing & evaluation

=over 4

=item idpdl

 $idPdl = $rel->idpdl($idPdl);
 $idPdl = $rel->idpdl(\@ids);
 $idPdl = $rel->idpdl($id);

Ensure L<PDL|PDL>-ness of a set of integer IDs.

=item tupleIds

 $tupleIds = $rel->tupleIds($attrType, $attrName, $valIds);

Returns a L<PDL|PDL> representing the set of index items
of type C<$attrType> whose value for the C<$attrName> attribute is
contained in the ID-set C<$valIds>, which may be specified
in any of the forms accepted by the L<idpdl()|/idpdl> method.

C<$attrType> is either 't' for a term-attribute
(in which case the returned C<$tupleIds> are term indices),
or 'm' for a metadata attribute
(in which case the returned C<$tupleIds> are "category" indices).
The returned C<$tupleIds> are always sorted in ascending order.

Could use some optimization.

=item termIds

 $ti = $rel->termIds($tattrName, $valIds);

wraps L<$rel-E<gt>tupleIds('t',$tattrName,$valIds)|/tupleIds>.

=item catIds

 $ci = $rel->catIds($mattrName, $valIds);

wraps L<$rel-E<gt>tupleIds('m',$mattrName,$valIds)|/tupleIds>.

=item hasMeta

 $bool = $rel->hasMeta($mattr);

returns true iff $rel supports metadata attribute $mattr.

=item metaEnum

 $enum_or_undef = $rel->metaEnum($mattr);

returns metadata attribute enum for $attr, or undef if $mattr is not supported.

=item catSubset

 $cats = $rel->catSubset($termIds);
 $cats = $rel->catSubset($termIds,$catIds)

Get a (sorted) cat-subset for the (sorted) term-set $termIds:
the set of all "categories" (original source files) which contain
at least one instance of any of the terms in $termIds,
optionally restricted to the (sorted and unique) set $catIds.
The returned category-IDs are sorted and unique.

=item groupby

 \%groupby = $rel->groupby($coldb, $groupby_request, %opts);
 \%groupby = $rel->groupby($coldb, \%groupby,        %opts);

Modified version of L<DiaColloDB::groupby()|DiaColloDB/groupby> suitable for pdl-ized TDF relation.
$grouby_request is as for L<DiaColloDB::parseRequest()|DiaColloDB/parseRequest>.
Returns a HASH-ref:

 ##-- COMPAT: equivalent to DiaColloDB::groupby() return values
 req => $request,    ##-- save request
 areqs => \@areqs,   ##-- parsed attribute requests ([$attr,$ahaving, \%ainfo],...)
                     ##   + new: %ainfo = ( aname=>$enum_name, atype=>$t_or_m, apos=>$apos )
 attrs => \@attrs,   ##-- like $coldb->attrs($groupby_request), modulo "having" parts
 titles => \@titles, ##-- like map {$coldb->attrTitle($_)} @attrs
 ##
 ##-- NEW: for DiaColloDB::Relation::TDF
 how      => $ghow,     ##-- one of  't':groupby terms-only, 'c':groupby cats-only, 'tc':groupby terms+cats
 gatype   => $gatype,   ##-- pdl ($NG)         : attribute types $ai : 0 if $areqs->[$ai] is a term attribute, 1 if meta-attribute
 gapos    => $gapos,    ##-- pdl ($NG)         : term- or meta-attribute position indices $ai : $rel->mpos($attrs[$ai]) or $rel->tpos($attrs[$ai])
 ghavingt => $ghavingt, ##-- pdl ($NHavingTOk) : term indices $ti s.t. $ti matches groupby "having" requests, or undef
 ghavingc => $ghavingc, ##-- pdl ($NHavingCOk) : cat  indices $ci s.t. $ci matches groupby "having" requests, or undef
 g2s      => \&g2s,     ##-- stringification object suitable for DiaColloDB::Profile::stringify() [CODE,enum, or undef]
 gpack    => $packas,   ##-- pack template for groupby-keys

%opts:

 warn  => $level,    ##-- log-level for unknown attributes (default: 'warn')
 relax => $bool,     ##-- allow unsupported attributes (default=0)

=back

=cut

##----------------------------------------------------------------
## DESCRIPTION: DiaColloDB::Relation::TDF: Relation API: default: query info
=pod

=head2 Relation API: default: query info

=over 4

=item qinfo

 \%qinfo = $rel->qinfo($coldb, %opts);


get query-info hash for profile administrivia (ddc hit links).
%opts: as for L<profile()|/profile> method.
returned hash \%qinfo should have keys:

 fcoef     => $fcoef,     ##-- frequency coefficient (constant 1 here)
 qtemplate => $qtemplate, ##-- query template with __W1.I1__ rsp __W2.I2__ replacing groupby fields

=back

=cut

##========================================================================
## END POD DOCUMENTATION, auto-generated by podextract.perl

##======================================================================
## Footer
##======================================================================
=pod

=head1 AUTHOR

Bryan Jurish E<lt>moocow@cpan.orgE<gt>

=head1 COPYRIGHT AND LICENSE

Copyright (C) 2015-2016 by Bryan Jurish

This package is free software; you can redistribute it and/or modify
it under the same terms as Perl itself, either Perl version 5.14.2 or,
at your option, any later version of Perl 5 you may have available.

=head1 SEE ALSO

L<DiaColloDB::Relation(3pm)|DiaColloDB::Relation>,
L<DiaColloDB::Relation::TDF::Query(3pm)|DiaColloDB::Relation::TDF::Query>,
L<DiaColloDB::Relation::Cofreqs(3pm)|DiaColloDB::Relation::Cofreqs>,
L<DiaColloDB::Relation::Unigrams(3pm)|DiaColloDB::Relation::Unigrams>,
L<DiaColloDB::Relation::DDC(3pm)|DiaColloDB::Relation::DDC>,
L<DiaColloDB(3pm)|DiaColloDB>,
L<perl(1)|perl>,
...



=cut
