<apidoc:module name="ClassifierBuiltins" category="Classifier" lib="cts" xmlns:apidoc="http://marklogic.com/xdmp/apidoc"><apidoc:summary access="public">
<p xmlns="http://www.w3.org/1999/xhtml">The classifier built-in functions perform automatic classification of 
documents using training data. The classifiers that result from 
training are represented in XML. The classifier APIs and the XML output
from <code><a href="#cts:train">cts:train</a></code> conform to the 
<code>classifier.xsd</code> schema, located in the Config directory 
under the directory in which MarkLogic Server is installed.</p>

</apidoc:summary><apidoc:function name="train" type="builtin" lib="cts" category="Classifier" hidden="false" bucket="MarkLogic Built-In Functions">
    <apidoc:summary>
  Produces a set of classifiers from a list of
  labeled training documents.
</apidoc:summary>
    <apidoc:params>
      <apidoc:param name="training-nodes" type="node()*" optional="false">
    The sequence of training nodes.  These are nodes that represent 
    members of the classes.
  </apidoc:param>
      <apidoc:param name="labels" type="element(cts:label)*" optional="false">
    A sequence of labels for the training nodes, in the order corresponding
    to the training nodes.
  </apidoc:param>
      <apidoc:param name="options" type="element()?" optional="true">
    An XML representation of the options for defining the training 
    parameters. The options node must be in the <code xmlns="http://www.w3.org/1999/xhtml">cts:train</code>
    namespace. The following is a sample options node:
    <pre xml:space="preserve" xmlns="http://www.w3.org/1999/xhtml">
    &lt;options xmlns="cts:train"&gt;
      &lt;classifier-type&gt;supports&lt;/classifier-type&gt;
      &lt;kernel&gt;geodesic&lt;/kernel&gt;
    &lt;/options&gt; </pre>
    
    <p xmlns="http://www.w3.org/1999/xhtml">The <code>cts:train</code> options include:</p>
    <dl xmlns="http://www.w3.org/1999/xhtml">
    <dt><p>&lt;<code>classifier-type</code>&gt;</p></dt>
    <dd>A string defining the kind of classifier to produce, either 
    <code>weights</code> or <code>supports</code>. The default is 
    <code>weights</code>.</dd>

    <dt><p>&lt;<code>kernel</code>&gt;</p></dt>
    <dd>A string defining which function to use for comparing documents. 
    The default is <code>sqrt</code>. Normalization (the values that end in
    <code>-normalized</code>) brings document vectors into the unit sphere, 
    which may improve the mathematical properties of the calculations. 
    Possible values are:
      <dl>
      <dt><p><code>simple</code></p></dt>
      <dd>Model documents as 1 or 0 for presence or absence of each term.</dd>
      <dt><p><code>simple-normalized</code></p></dt>
      <dd>Like <code>simple</code>, but normalized by square root of 
      document length.</dd>
      <dt><p><code>sqrt</code></p></dt>
      <dd>Model documents using the square root of the term frequencies.</dd>
      <dt><p><code>sqrt-normalized</code></p></dt>
      <dd>Like <code>sqrt</code>, but normalized by the sum of the term 
      frequencies.</dd>
      <dt><p><code>linear-normalized</code></p></dt>
      <dd>Model documents as the term frequencies normalized by the 
      square root of the sum of the squares of the term frequencies.</dd>
      <dt><p><code>gaussian</code></p></dt>
      <dd>Compare documents using the Gaussian of the term frequencies.
      Requires a <code>classifier-type</code> of <code>supports</code>.</dd>
      <dt><p><code>geodesic</code></p></dt>
      <dd>Compare documents using the Riemann geodesic distance over 
      term frequencies. Requires a <code>classifier-type</code> of 
      <code>supports</code>.</dd>
      </dl>
    </dd>
    <dt><p>&lt;<code>max-terms</code>&gt;</p></dt>
    <dd>An integer defining the maximum number of terms to use to
    represent each document.  If a positive number M is given, then the
    M most discriminating terms are used; other terms are dropped.  The 
    default is 0 (unlimited).</dd>
    <dt><p>&lt;<code>max-support</code>&gt;</p></dt>
    <dd>A double specifying the maximum influence a single training node 
    can have.  This parameter has a strong influence on performance. 
    The default value of 1.0 should work well in most cases. Larger 
    values means greater sensitivity and may improve accuracy on small 
    datasets, but give longer running times.  Smaller values mean less 
    sensitivity and better resistance to mis-classified documents, and 
    shorter running times.</dd>
    <dt><p>&lt;<code>min-weight</code>&gt;</p></dt>
    <dd>A double specifying the minimum weight a term can have and still
    be considered for inclusion in the term vector. This parameter only applies
    to the term weight form of the classifier. Smaller values mean longer
    term vectors and as a consequence longer running times and greater memory
    consumption during classification, but may also improve accuracy.
    The default is is 0.01.</dd>
    <dt><p>&lt;<code>tolerance</code>&gt;</p></dt>
    <dd>How close the final solutions to the constraint equations must be.  
    Smaller values lead to a greater number of iterations and longer 
    running times.  Larger values lead to less precise classification.  
    The default is 0.01.</dd>
    <dt><p>&lt;<code>epsilon</code>&gt;</p></dt>
    <dd>How close a value must be to 0 to be counted as equal to 0.  
    Since double arithmetic is not precise, setting this value to exactly 
    0 will likely lead to non-convergence of the algorithm.  Smaller 
    values lead to a greater number of iterations and longer running 
    times.  Larger values lead to less precise classification.  
    The default is 0.01.</dd>
    <dt><p>&lt;<code>max-iterations</code>&gt;</p></dt>
    <dd>The maximum number of iterations of the constraint satisfaction 
    algorithm to run. The algorithm usually converges very quickly, 
    so this parameter usually has no effect unless it is set very low.  
    The default is 500.</dd>
    <dt><p><code>&lt;thresholds&gt;</code></p></dt>
    <dd>A definition of the thresholds to use in classification. This is
    a complex element with one or more <code>&lt;threshold&gt;</code> 
    children.  You can specify both a global value and per-class values 
    (as computed from <code>cts:thresholds</code>).  The global value 
    will apply to any classes for which a per-class value is not 
    specified.  For example: 
    <pre xml:space="preserve">
    &lt;options xmlns="cts:train"&gt;
      &lt;thresholds&gt;
        &lt;threshold&gt;-1.0&lt;/threshold&gt;
        &lt;threshold class="Example 1"&gt;-2.42&lt;/threshold&gt;
      &lt;/thresholds&gt;
    &lt;/options&gt;
    </pre>
    <p>For the initial tuning phase of training your data, leave the value
    of this parameter at its default value which is a very large negative
    number (-10E30).  This will allow you to accurately compute the
    threshold values when you run <code>cts:thresholds</code> on the initial
    training data.  Then you can use the calculated thresholds values  
    when you run the secondary pass through the second part of your training 
    data.</p>
    </dd>
    <dt><p>&lt;<code>use-db-config</code>&gt;</p></dt>
    <dd>A boolean value indicating whether to use the current DB configuration 
    for determining which terms to use.  The default is <code>false</code>,
    which means that only the indexing options in the options node will be 
    used for calculating the classifier.
    </dd>

    </dl>
    <p xmlns="http://www.w3.org/1999/xhtml">The options element also includes indexing options in the 
    <code>http://marklogic.com/xdmp/database</code> namespace.  
    These control which terms to use. Note that the use of certain 
    options, such as <code>fast-case-sensitive-searches</code>, will not 
    impact final results unless the term vector size is limited with 
    the <code>max-terms</code> option.  Other options, such as 
    <code>phrase-throughs</code>, will only generate terms if some 
    other option is also enabled (in this case 
    <code>fast-phrase-searches</code>). </p>
    <p xmlns="http://www.w3.org/1999/xhtml">The database options are the same as the database options shown for
    <a href="cts:distinctive-terms#db-term-options">
    <code>cts:distinctive-terms</code></a>.</p>

  </apidoc:param>
    </apidoc:params>
    <apidoc:return>element(cts:classifier)?</apidoc:return>
    <apidoc:usage>
<p xmlns="http://www.w3.org/1999/xhtml">
The elements in the label sequence should match one for one with the nodes
in the training node sequence. The first label element describes the first node
in the training node sequence, the second label element describes the second
node in the training node sequence, and so on. 
If there are more labels than training nodes or more training nodes
than labels, an error is raised.
</p>
<p xmlns="http://www.w3.org/1999/xhtml">
The format of each label element is:
</p>
<pre xml:space="preserve" xmlns="http://www.w3.org/1999/xhtml">
  &lt;cts:label name="Node1"&gt;
    &lt;cts:class name="Example1"/&gt;
    &lt;cts:class name="Example2" val="-1"/&gt;
        :   :
  &lt;/cts:label&gt;
</pre>
<p xmlns="http://www.w3.org/1999/xhtml">Each class listed indicates whether the corresponding node in the training
sequence is in the given class. Examples are taken to be positive examples
unless specified otherwise (with a <code>val</code> attribute of -1).  
The document is assumed to be a negative example of any classes that are
not explicitly listed. 
The name attribute on the label element is an optional name for the labelled 
node. It is purely for human consumption to help in tuning the classification 
parameters.
</p>

<a id="outputformats" xmlns="http://www.w3.org/1999/xhtml"><b>Output Formats</b></a>

<p xmlns="http://www.w3.org/1999/xhtml">A linear classifier is defined by a weight vector w on terms, and
an offset value b. The &lt;weights/&gt; node encodes the weight vector
directly. Its children are the classes, and each class includes
a list of terms. The term node uses an internal id to identify the term
and a term weight:</p>

<pre xml:space="preserve" xmlns="http://www.w3.org/1999/xhtml">
&lt;weights&gt;
  &lt;class name="Example1" offset="2.04"&gt;
    &lt;term id="43587329645324245" val="0.3423432"/&gt;
    &lt;term id="47893427895432534" val="-0.12345556"/&gt;
      :                           :
  &lt;/class&gt;
      :
&lt;/weights&gt;
</pre>
<p xmlns="http://www.w3.org/1999/xhtml">     
The weight vector w is a linear combination of the documents
themselves, and it may be more convenient to express the classifier in
this way. For instance, if the number of terms is not limited, the
&lt;weights/&gt; node will be extremely large. The weight vector form
may not be used if the classifier kernel is non-linear, that is, with
the Gaussian or geodesic kernel. 
</p>
<p xmlns="http://www.w3.org/1999/xhtml">The support vector representation of the classifier includes a
supports node that has &lt;class/&gt; children for each class. Here the
class elements contain a list of doc elements which identify the specific
training nodes using an internal key. This internal key is valid across
queries only for nodes in the database.  Each doc element has an 
attribute encoding the weight of that document and an error attribute
which shows how well the document fit the classifier. Large positive
or negative errors (greater than about 1.5) are potentially
mis-classified documents.</p>

<pre xml:space="preserve" xmlns="http://www.w3.org/1999/xhtml">
&lt;supports&gt;
  &lt;class name="Example1" offset="2.04"&gt;
    &lt;doc id="155584958759" name="Node102" val="-0.00334163" err="1.4"/&gt;
    &lt;doc id="594064848864" name="Node57" val="0.025341234" err="-2.3"/&gt;
      :                             :
  &lt;/class&gt;
      :    
&lt;/supports&gt;
</pre>
<p xmlns="http://www.w3.org/1999/xhtml">Each class is identified by a unique name.</p>
</apidoc:usage>
    <apidoc:example><a id="trainEx1" xmlns="http://www.w3.org/1999/xhtml"></a><pre xml:space="preserve" xmlns="http://www.w3.org/1999/xhtml">
let $firsthalf := xdmp:directory("/shakespeare/plays/", "1")[1 to 19]
let $labels := for $x in $firsthalf 
         return
         &lt;cts:label&gt;
           &lt;cts:class name={xdmp:document-properties(xdmp:node-uri($x))
                //playtype/text()}/&gt;
          &lt;/cts:label&gt;
return
cts:train($firsthalf, $labels, 
       &lt;options xmlns="cts:train"&gt;
         &lt;classifier-type&gt;supports&lt;/classifier-type&gt;
       &lt;/options&gt;)

  =&gt;  &lt;cts:classifier&gt;...

</pre></apidoc:example>
    <apidoc:example><a id="trainEx2" xmlns="http://www.w3.org/1999/xhtml"></a><pre xml:space="preserve" xmlns="http://www.w3.org/1999/xhtml">
xquery version "1.0-ml";

(: 
   this is the same as the previous example, except it uses the 
   use-db-config option
:)
let $firsthalf := 
  xdmp:directory("/shakespeare/plays/", "1")[1 to 19]
let $labels := for $x in $firsthalf 
         return
         &lt;cts:label&gt;
           &lt;cts:class name={xdmp:document-properties(
                                       xdmp:node-uri($x))
                //playtype/text()}/&gt;
          &lt;/cts:label&gt;
return
cts:train($firsthalf, $labels, 
       &lt;options xmlns="cts:train"&gt;
         &lt;classifier-type&gt;supports&lt;/classifier-type&gt;
         &lt;use-db-config&gt;true&lt;/use-db-config&gt;
       &lt;/options&gt;)

  =&gt;  &lt;cts:classifier&gt;...

</pre></apidoc:example>
  </apidoc:function><apidoc:function name="classify" type="builtin" lib="cts" category="Classifier" hidden="false" bucket="MarkLogic Built-In Functions">
    <apidoc:summary>
  Classifies a sequence of nodes based on training data.  The training data
  is in the form of a classifier specification, which is generated from the 
  output of <code xmlns="http://www.w3.org/1999/xhtml">cts:train</code>. Returns labels for each of the input
  documents in the same order as the input document.
</apidoc:summary>
    <apidoc:params>
      <apidoc:param name="data-nodes" type="node()*" optional="false">
 The sequence of nodes to be classified.  
 </apidoc:param>
      <apidoc:param name="classifier" type="element(cts:classifier)" optional="false">
 An element node containing the classifier specification.  This is typically
 the output of <code xmlns="http://www.w3.org/1999/xhtml">cts:train</code>, either run directly or saved in an
 XML document in the database. 
 </apidoc:param>
      <apidoc:param name="options" type="element()?" optional="false">
  <p xmlns="http://www.w3.org/1999/xhtml">An options element.  The options for classification are passed 
  automatically from <code>cts:train</code> to the <code>cts:classifier</code>
  specification as part of the classifier element so that they are 
  consistent with the parameters used in training.  The following option
  may be separately passed to <code>cts:classify</code> and is in the 
  <code>cts:classify</code> namespace:</p>
   <dl xmlns="http://www.w3.org/1999/xhtml">
   <dt><p><code>&lt;thresholds&gt;</code></p></dt>
   <dd>A definition of the thresholds to use in classification. This is
   a complex element with one or more <code>&lt;threshold&gt;</code> children.  
   You can specify both a global value and per-class values (as computed from 
   <code>cts:thresholds</code>).  The global value will apply to 
   any classes for which a per-class value is not specified.  For example: 
   <pre xml:space="preserve">
   &lt;options xmlns="cts:classify"&gt;
     &lt;thresholds&gt;
       &lt;threshold&gt;-1.0&lt;/threshold&gt;
       &lt;threshold class="Example 1"&gt;-2.42&lt;/threshold&gt;
     &lt;/thresholds&gt;
   &lt;/options&gt;
   </pre>
   </dd>
   </dl>
 </apidoc:param>
      <apidoc:param name="training-nodes" type="node()*" optional="false">
 The sequence of training nodes used to train the classifier. 
 Required if the <code xmlns="http://www.w3.org/1999/xhtml">supports</code> form of the classifier is used; 
 ignored if the <code xmlns="http://www.w3.org/1999/xhtml">weights</code> form of the classifier is used.
 </apidoc:param>
    </apidoc:params>
    <apidoc:return>element(cts:label)*</apidoc:return>
    <apidoc:usage> 
  <p xmlns="http://www.w3.org/1999/xhtml"><code>cts:classify</code> classifies a sequence of nodes using
  the output from <code>cts:train</code>. The <code>$data-nodes</code>
  and <code>$classifier</code> parameters are respectively the nodes to 
  be classified and the specification output from <code>cts:train</code>.  
  <code>cts:classify</code> can use either <code>supports</code> or
  <code>weights</code> forms of the <code>$classifier</code> output 
  from <code>cts:train</code> (see <a href="#outputformats">Output 
  Formats</a>).  If the <code>supports</code> form is used, the training 
  nodes must be passed as the 4th parameter.  The <code>$options</code>
  parameter is an options element in the <code>cts:classify</code> namespace.
</p>

<p xmlns="http://www.w3.org/1999/xhtml">
The output is a sequence of label elements of the form:
</p>
<pre xml:space="preserve" xmlns="http://www.w3.org/1999/xhtml">
&lt;cts:label&gt; 
  &lt;cts:class name="Example 1" val="-0.003"/&gt;
  &lt;cts:class name="Example 2" val="1.4556"/&gt;
  ...
&lt;/cts:label&gt;
</pre>
  <p xmlns="http://www.w3.org/1999/xhtml">Each label corresponds to the data node in the corresponding position
  in the input sequence. There will be a <code>&lt;class&gt;</code> child 
  for each class where the document passed the class threshold. The 
  <code>val</code> attribute gives the class membership value for the 
  data node in the given class.  Values greater than zero indicate 
  likely class membership, values less than zero indicate likely 
  non-membership.  Adjusting thresholds can give more or less selective 
  classification. Increasing the threshold leads to a more selective
  classification (that is, decreases the likelihood of classification in the
  class). Decreasing the threshold gives less selective classification.
  </p>
</apidoc:usage>
    <apidoc:example><pre xml:space="preserve" xmlns="http://www.w3.org/1999/xhtml">
let $firsthalf := xdmp:directory("/shakespeare/plays/", "1")[1 to 19]
let $secondhalf := xdmp:directory("/shakespeare/plays/", "1")[20 to 37]
let $classifier :=  
  let $labels := for $x in $firsthalf 
         return
         &lt;cts:label&gt;
           &lt;cts:class name={xdmp:document-properties(xdmp:node-uri($x))
                 //playtype/text()}/&gt;
         &lt;/cts:label&gt;
  return
  cts:train($firsthalf, $labels, 
          &lt;options xmlns="cts:train"&gt;
            &lt;classifier-type&gt;supports&lt;/classifier-type&gt;
          &lt;/options&gt;)
return
cts:classify($secondhalf, $classifier, 
             &lt;options xmlns="cts:classify"/&gt;,
             $firsthalf)

  =&gt; ( &lt;label&gt;...&lt;/label&gt;,... )

</pre></apidoc:example>
  </apidoc:function><apidoc:function name="thresholds" type="builtin" lib="cts" category="Classifier" hidden="false" bucket="MarkLogic Built-In Functions">
    <apidoc:summary>
  Compute precision, recall, the F measure, and thresholds for the 
  classes computed by the classifier, by comparing with the labels 
  for the same set. 
</apidoc:summary>
    <apidoc:params>
      <apidoc:param name="computed-labels" type="element(cts:label)*" optional="false"> 
 A sequence of element nodes containing the labels from classification 
 (the output from <code xmlns="http://www.w3.org/1999/xhtml">cts:classify</code>) for a set of documents.
 </apidoc:param>
      <apidoc:param name="known-labels" type="element(cts:label)*" optional="false"> 
 A sequence of element nodes containing the known labels for the same set 
 of documents.
 </apidoc:param>
      <apidoc:param name="recall-weight" type="xs:double?" optional="true"> 
 The factor to use in the calculation of the F measure. The number should 
 be non-negative. A value of 0 means F is just precision and a value 
 of +INF means F is just recall. The default is 1, which gives the harmonic 
 mean between precision and recall.
 </apidoc:param>
    </apidoc:params>
    <apidoc:return>element(cts:thresholds)?</apidoc:return>
    <apidoc:usage> 
  <p xmlns="http://www.w3.org/1999/xhtml">You use the output of <code>cts:thresholds</code> to determine the best 
  thresholds values for your data, based on the first pass through the first
  part of your training data.  The output of <code>cts:thresholds</code> 
  provides you with precision and recall measurements at the calculated 
  thresholds for each class.  The following are the definitions of the 
  attributes of the <code>thresholds</code> element returned 
  by <code>cts:thresholds</code>:</p>
  <dl xmlns="http://www.w3.org/1999/xhtml">
  <dt><p><code>name</code></p></dt>
  <dd>The name of the class.</dd>
  <dt><p><code>threshold</code></p></dt>
  <dd>The threshold that is computed by the classifier to give the best 
  results.  The threshold is used by <code>cts:classify</code> when
  classifying documents, and is defined to be the positive 
  or negative distance from the hyperplane which represents the edge of 
  the class.
  </dd>
  <dt><p><code>precision</code></p></dt>
  <dd> A number which represents the fraction of nodes identified in a 
  class that are actually in that class.  As this aproaches 1, there is 
  a higher probability that you over-classified.</dd>
  <dt><p><code>recall</code></p></dt>
  <dd>A number which represents the the fraction of nodes in a class that
  were identified by the classifier as being in that class. As this 
  aproaches 1, there is a higher probability that you under-classified.</dd>
  <dt><p><code>F</code> (the F-measure)</p></dt>
  <dd>A measure which represents if the classification at the given
  threshold is closer to recall or closer to precision.  A value of 1
  indicates that precision and recall have equal weight.  A value of 0.5 
  indicates that precision is weighted 2x recall.  A value of 2 indicates
  that recall is weighted 2x prcision.  A value of 0 indicates that the
  weighting is precision only, and a value of +INF 
  (<code>xs:double('+INF')</code>) indicates that weighting is recall only.</dd>
  </dl>
</apidoc:usage>
    <apidoc:example><pre xml:space="preserve" xmlns="http://www.w3.org/1999/xhtml">
let $firsthalf := xdmp:directory("/shakespeare/plays/", "1")[1 to 19]
let $secondhalf := xdmp:directory("/shakespeare/plays/", "1")[20 to 37]
let $firstlabels := for $x in $firsthalf 
        return
        &lt;cts:label&gt;
          &lt;cts:class name={xdmp:document-properties(xdmp:node-uri($x))
                                     //playtype/text()}/&gt;
        &lt;/cts:label&gt;
let $secondlabels := for $x in $secondhalf 
        return
        &lt;cts:label&gt;
          &lt;cts:class name={xdmp:document-properties(xdmp:node-uri($x))
                                     //playtype/text()}/&gt;
        &lt;/cts:label&gt;
let $classifier :=  
    cts:train($firsthalf, $firstlabels, 
      &lt;options xmlns="cts:train"&gt;
        &lt;classifier-type&gt;supports&lt;/classifier-type&gt;
      &lt;/options&gt;)
let $classifysecond :=
  cts:classify($secondhalf, $classifier, 
        &lt;options xmlns="cts:classify"/&gt;,
        $firsthalf)
return
cts:thresholds($classifysecond, $secondlabels)
(: 
   This returns the computed thresholds for the second half of 
   the plays in a Shakespeare database, based on a classifier
   trained with the first half of the plays.  For example:

&lt;thresholds xmlns="http://marklogic.com/cts"&gt;
  &lt;class name="TRAGEDY" threshold="0.221948" precision="1" 
         recall="0.666667" f="0.8" count="3"/&gt;
  &lt;class name="COMEDY" threshold="0.114389" precision="0.916667" 
         recall="1" f="0.956522" count="11"/&gt;
  &lt;class name="HISTORY" threshold="0.567648" precision="1" 
         recall="1" f="1" count="4"/&gt;
&lt;/thresholds&gt;
:)
</pre></apidoc:example>
  </apidoc:function></apidoc:module>