/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License. You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package opennlp.tools.cmdline;

import java.io.OutputStream;
import java.io.PrintStream;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;

import opennlp.tools.util.Span;
import opennlp.tools.util.eval.FMeasure;
import opennlp.tools.util.eval.Mean;

public abstract class FineGrainedReportListener {

  private static final char[] alpha = { 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h',
      'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v',
      'w', 'x', 'y', 'z' };
  private final PrintStream printStream;
  private final Stats stats = new Stats();

  public FineGrainedReportListener(PrintStream printStream) {
    this.printStream = printStream;
  }

  /**
   * Writes the report to the {@link OutputStream}. Should be called only after
   * the evaluation process.
   */
  public FineGrainedReportListener(OutputStream outputStream) {
    this.printStream = new PrintStream(outputStream);
  }

  private static String generateAlphaLabel(int index) {

    char[] labelChars = new char[3];
    int i;

    for (i = 2; i >= 0; i--) {
      if (index >= 0) {
        labelChars[i] = alpha[index % alpha.length];
        index = index / alpha.length - 1;
      } else {
        labelChars[i] = ' ';
      }
    }
    return new String(labelChars);
  }

  public abstract void writeReport();


  // api methods
  // general stats

  protected Stats getStats() {
    return this.stats;
  }

  private long getNumberOfSentences() {
    return stats.getNumberOfSentences();
  }

  private double getAverageSentenceSize() {
    return stats.getAverageSentenceSize();
  }

  private int getMinSentenceSize() {
    return stats.getMinSentenceSize();
  }

  private int getMaxSentenceSize() {
    return stats.getMaxSentenceSize();
  }

  private int getNumberOfTags() {
    return stats.getNumberOfTags();
  }

  // token stats

  private double getAccuracy() {
    return stats.getAccuracy();
  }

  private double getTokenAccuracy(String token) {
    return stats.getTokenAccuracy(token);
  }

  private SortedSet<String> getTokensOrderedByFrequency() {
    return stats.getTokensOrderedByFrequency();
  }

  private int getTokenFrequency(String token) {
    return stats.getTokenFrequency(token);
  }

  private int getTokenErrors(String token) {
    return stats.getTokenErrors(token);
  }

  private SortedSet<String> getTokensOrderedByNumberOfErrors() {
    return stats.getTokensOrderedByNumberOfErrors();
  }

  private SortedSet<String> getTagsOrderedByErrors() {
    return stats.getTagsOrderedByErrors();
  }

  private int getTagFrequency(String tag) {
    return stats.getTagFrequency(tag);
  }

  private int getTagErrors(String tag) {
    return stats.getTagErrors(tag);
  }

  private double getTagPrecision(String tag) {
    return stats.getTagPrecision(tag);
  }

  private double getTagRecall(String tag) {
    return stats.getTagRecall(tag);
  }

  private double getTagFMeasure(String tag) {
    return stats.getTagFMeasure(tag);
  }

  private SortedSet<String> getConfusionMatrixTagset() {
    return stats.getConfusionMatrixTagset();
  }

  private SortedSet<String> getConfusionMatrixTagset(String token) {
    return stats.getConfusionMatrixTagset(token);
  }

  private double[][] getConfusionMatrix() {
    return stats.getConfusionMatrix();
  }

  private double[][] getConfusionMatrix(String token) {
    return stats.getConfusionMatrix(token);
  }

  private String matrixToString(SortedSet<String> tagset, double[][] data, boolean filter) {
    // we don't want to print trivial cases (acc=1)
    int initialIndex = 0;
    String[] tags = tagset.toArray(new String[0]);
    StringBuilder sb = new StringBuilder();
    int minColumnSize = Integer.MIN_VALUE;
    String[][] matrix = new String[data.length][data[0].length];
    for (int i = 0; i < data.length; i++) {
      int j = 0;
      for (; j < data[i].length - 1; j++) {
        matrix[i][j] = data[i][j] > 0 ? Integer.toString((int) data[i][j])
            : ".";
        if (minColumnSize < matrix[i][j].length()) {
          minColumnSize = matrix[i][j].length();
        }
      }
      matrix[i][j] = MessageFormat.format("{0,number,#.##%}", data[i][j]);
      if (data[i][j] == 1 && filter) {
        initialIndex = i + 1;
      }
    }

    final String headerFormat = "%" + (minColumnSize + 2) + "s "; // | 1234567 |
    final String cellFormat = "%" + (minColumnSize + 2) + "s "; // | 12345 |
    final String diagFormat = " %" + (minColumnSize + 2) + "s";
    for (int i = initialIndex; i < tagset.size(); i++) {
      sb.append(String.format(headerFormat,
          generateAlphaLabel(i - initialIndex).trim()));
    }
    sb.append("| Accuracy | <-- classified as\n");
    for (int i = initialIndex; i < data.length; i++) {
      int j = initialIndex;
      for (; j < data[i].length - 1; j++) {
        if (i == j) {
          String val = "<" + matrix[i][j] + ">";
          sb.append(String.format(diagFormat, val));
        } else {
          sb.append(String.format(cellFormat, matrix[i][j]));
        }
      }
      sb.append(
          String.format("|   %-6s |   %3s = ", matrix[i][j],
              generateAlphaLabel(i - initialIndex))).append(tags[i]);
      sb.append("\n");
    }
    return sb.toString();
  }

  protected void printGeneralStatistics() {
    printHeader("Evaluation summary");
    printStream.append(
        String.format("%21s: %6s", "Number of sentences", getNumberOfSentences())).append("\n");
    printStream.append(
        String.format("%21s: %6s", "Min sentence size", getMinSentenceSize())).append("\n");
    printStream.append(
        String.format("%21s: %6s", "Max sentence size", getMaxSentenceSize())).append("\n");
    printStream.append(
        String.format("%21s: %6s", "Average sentence size",
            MessageFormat.format("{0,number,#.##}", getAverageSentenceSize()))).append("\n");
    printStream.append(
        String.format("%21s: %6s", "Tags count", getNumberOfTags())).append("\n");
    printStream.append(
        String.format("%21s: %6s", "Accuracy",
            MessageFormat.format("{0,number,#.##%}", getAccuracy()))).append("\n");
    printFooter("Evaluation Corpus Statistics");
  }

  protected void printTokenOccurrencesRank() {
    printHeader("Most frequent tokens");

    SortedSet<String> toks = getTokensOrderedByFrequency();
    final int maxLines = 20;

    int maxTokSize = 5;

    int count = 0;
    Iterator<String> tokIterator = toks.iterator();
    while (tokIterator.hasNext() && count++ < maxLines) {
      String tok = tokIterator.next();
      if (tok.length() > maxTokSize) {
        maxTokSize = tok.length();
      }
    }

    int tableSize = maxTokSize + 19;
    String format = "| %3s | %6s | %" + maxTokSize + "s |";

    printLine(tableSize);
    printStream.append(String.format(format, "Pos", "Count", "Token")).append("\n");
    printLine(tableSize);

    // get the first 20 errors
    count = 0;
    tokIterator = toks.iterator();
    while (tokIterator.hasNext() && count++ < maxLines) {
      String tok = tokIterator.next();
      int frequency = getTokenFrequency(tok);

      printStream.append(String.format(format, count, frequency, tok)

      ).append("\n");
    }
    printLine(tableSize);
    printFooter("Most frequent tokens");
  }

  protected void printTokenErrorRank() {
    printHeader("Tokens with the highest number of errors");
    printStream.append("\n");

    SortedSet<String> toks = getTokensOrderedByNumberOfErrors();
    int maxTokenSize = 5;

    int count = 0;
    Iterator<String> tokIterator = toks.iterator();
    while (tokIterator.hasNext() && count++ < 20) {
      String tok = tokIterator.next();
      if (tok.length() > maxTokenSize) {
        maxTokenSize = tok.length();
      }
    }

    int tableSize = 31 + maxTokenSize;

    String format = "| %" + maxTokenSize + "s | %6s | %5s | %7s |\n";

    printLine(tableSize);
    printStream.append(String.format(format, "Token", "Errors", "Count", "% Err"));
    printLine(tableSize);

    // get the first 20 errors
    count = 0;
    tokIterator = toks.iterator();
    while (tokIterator.hasNext() && count++ < 20) {
      String tok = tokIterator.next();
      int ocurrencies = getTokenFrequency(tok);
      int errors = getTokenErrors(tok);
      String rate = MessageFormat.format("{0,number,#.##%}", (double) errors
          / ocurrencies);

      printStream.append(String.format(format, tok, errors, ocurrencies, rate)

      );
    }
    printLine(tableSize);
    printFooter("Tokens with the highest number of errors");
  }

  protected void printTagsErrorRank() {
    printHeader("Detailed Accuracy By Tag");
    SortedSet<String> tags = getTagsOrderedByErrors();
    printStream.append("\n");

    int maxTagSize = 3;

    for (String t : tags) {
      if (t.length() > maxTagSize) {
        maxTagSize = t.length();
      }
    }

    int tableSize = 65 + maxTagSize;

    String headerFormat = "| %" + maxTagSize
        + "s | %6s | %6s | %7s | %9s | %6s | %9s |\n";
    String format = "| %" + maxTagSize
        + "s | %6s | %6s | %-7s | %-9s | %-6s | %-9s |\n";

    printLine(tableSize);
    printStream.append(String.format(headerFormat, "Tag", "Errors", "Count",
        "% Err", "Precision", "Recall", "F-Measure"));
    printLine(tableSize);

    for (String tag : tags) {
      int ocurrencies = getTagFrequency(tag);
      int errors = getTagErrors(tag);
      String rate = MessageFormat.format("{0,number,#.###}", (double) errors
          / ocurrencies);

      double p = getTagPrecision(tag);
      double r = getTagRecall(tag);
      double f = getTagFMeasure(tag);

      printStream.append(String.format(format, tag, errors, ocurrencies, rate,
          MessageFormat.format("{0,number,#.###}", p > 0 ? p : 0),
          MessageFormat.format("{0,number,#.###}", r > 0 ? r : 0),
          MessageFormat.format("{0,number,#.###}", f > 0 ? f : 0))

      );
    }
    printLine(tableSize);

    printFooter("Tags with the highest number of errors");
  }

  protected void printGeneralConfusionTable() {
    printHeader("Confusion matrix");

    SortedSet<String> labels = getConfusionMatrixTagset();

    double[][] confusionMatrix = getConfusionMatrix();

    printStream.append("\nTags with 100% accuracy: ");
    int line = 0;
    for (String label : labels) {
      if (confusionMatrix[line][confusionMatrix[0].length - 1] == 1) {
        printStream.append(label).append(" (")
            .append(Integer.toString((int) confusionMatrix[line][line]))
            .append(") ");
      }
      line++;
    }

    printStream.append("\n\n");

    printStream.append(matrixToString(labels, confusionMatrix, true));

    printFooter("Confusion matrix");
  }

  protected void printDetailedConfusionMatrix() {
    printHeader("Confusion matrix for tokens");
    printStream.append("  sorted by number of errors\n");
    SortedSet<String> toks = getTokensOrderedByNumberOfErrors();

    for (String t : toks) {
      double acc = getTokenAccuracy(t);
      if (acc < 1) {
        printStream
            .append("\n[")
            .append(t)
            .append("]\n")
            .append(
                String.format("%12s: %-8s", "Accuracy",
                    MessageFormat.format("{0,number,#.##%}", acc)))
            .append("\n");
        printStream.append(
            String.format("%12s: %-8s", "Ocurrencies",
                    getTokenFrequency(t))).append("\n");
        printStream.append(
            String.format("%12s: %-8s", "Errors",
                    getTokenErrors(t))).append("\n");

        SortedSet<String> labels = getConfusionMatrixTagset(t);

        double[][] confusionMatrix = getConfusionMatrix(t);

        printStream.append(matrixToString(labels, confusionMatrix, false));
      }
    }
    printFooter("Confusion matrix for tokens");
  }

  /** Auxiliary method that prints a emphasised report header */
  private void printHeader(String text) {
    printStream.append("=== ").append(text).append(" ===\n");
  }

  /** Auxiliary method that prints a marker to the end of a report */
  private void printFooter(String text) {
    printStream.append("\n<-end> ").append(text).append("\n\n");
  }

  /** Auxiliary method that prints a horizontal line of a given size */
  private void printLine(int size) {
    for (int i = 0; i < size; i++) {
      printStream.append("-");
    }
    printStream.append("\n");
  }

  /**
   * A comparator that sorts the confusion matrix labels according to the
   * accuracy of each line
   */
  public static class MatrixLabelComparator implements Comparator<String> {

    private final Map<String, ConfusionMatrixLine> confusionMatrix;

    public MatrixLabelComparator(Map<String, ConfusionMatrixLine> confusionMatrix) {
      this.confusionMatrix = confusionMatrix;
    }

    @Override
    public int compare(String o1, String o2) {
      if (o1.equals(o2)) {
        return 0;
      }
      ConfusionMatrixLine t1 = confusionMatrix.get(o1);
      ConfusionMatrixLine t2 = confusionMatrix.get(o2);
      if (t1 == null || t2 == null) {
        if (t1 == null) {
          return 1;
        } else {
          return -1;
        }
      }
      double r1 = t1.getAccuracy();
      double r2 = t2.getAccuracy();
      if (r1 == r2) {
        return o1.compareTo(o2);
      }
      if (r2 > r1) {
        return 1;
      }
      return -1;
    }
  }

  public static class GroupedMatrixLabelComparator implements Comparator<String> {

    private final HashMap<String, Double> categoryAccuracy;
    private final Map<String, ConfusionMatrixLine> confusionMatrix;

    public GroupedMatrixLabelComparator(Map<String, ConfusionMatrixLine> confusionMatrix) {
      this.confusionMatrix = confusionMatrix;
      this.categoryAccuracy = new HashMap<>();

      // compute grouped categories
      for (Entry<String, ConfusionMatrixLine> entry : confusionMatrix.entrySet()) {
        final String key = entry.getKey();
        final ConfusionMatrixLine confusionMatrixLine = entry.getValue();
        final String category;
        if (key.contains("-")) {
          category = key.split("-")[0];
        } else {
          category = key;
        }
        double currentAccuracy = categoryAccuracy.getOrDefault(category, 0.0d);
        categoryAccuracy.put(category, currentAccuracy + confusionMatrixLine.getAccuracy());
      }
    }

    @Override
    public int compare(String o1, String o2) {
      if (o1.equals(o2)) {
        return 0;
      }
      String c1 = o1;
      String c2 = o2;

      if (o1.contains("-")) {
        c1 = o1.split("-")[0];
      }
      if (o2.contains("-")) {
        c2 = o2.split("-")[0];
      }

      if (c1.equals(c2)) { // same category - sort by confusion matrix

        ConfusionMatrixLine t1 = confusionMatrix.get(o1);
        ConfusionMatrixLine t2 = confusionMatrix.get(o2);
        if (t1 == null || t2 == null) {
          if (t1 == null) {
            return 1;
          } else {
            return -1;
          }
        }
        double r1 = t1.getAccuracy();
        double r2 = t2.getAccuracy();
        if (r1 == r2) {
          return o1.compareTo(o2);
        }
        if (r2 > r1) {
          return 1;
        }
        return -1;
      } else { // different category - sort by category
        Double t1 = categoryAccuracy.get(c1);
        Double t2 = categoryAccuracy.get(c2);
        if (t1 == null || t2 == null) {
          if (t1 == null) {
            return 1;
          } else {
            return -1;
          }
        }
        if (t1.equals(t2)) {
          return o1.compareTo(o2);
        }
        if (t2 > t1) {
          return 1;
        }
        return -1;
      }
    }
  }

  public Comparator<String> getMatrixLabelComparator(Map<String, ConfusionMatrixLine> confusionMatrix) {
    return new MatrixLabelComparator(confusionMatrix);
  }

  public static class SimpleLabelComparator implements Comparator<String> {

    private final Map<String, Counter> map;

    public SimpleLabelComparator(Map<String, Counter> map) {
      this.map = map;
    }

    @Override
    public int compare(String o1, String o2) {
      if (o1.equals(o2)) {
        return 0;
      }
      int e1 = 0, e2 = 0;
      if (map.containsKey(o1))
        e1 = map.get(o1).value();
      if (map.containsKey(o2))
        e2 = map.get(o2).value();
      if (e1 == e2) {
        return o1.compareTo(o2);
      }
      return e2 - e1;
    }
  }

  public Comparator<String> getLabelComparator(Map<String, Counter> map) {
    return new SimpleLabelComparator(map);
  }

  public static class GroupedLabelComparator implements Comparator<String> {

    private final HashMap<String, Integer> categoryCounter;
    private final Map<String, Counter> labelCounter;

    public GroupedLabelComparator(Map<String, Counter> map) {
      this.labelCounter = map;
      this.categoryCounter = new HashMap<>();

      // compute grouped categories
      for (Entry<String, Counter> entry : labelCounter.entrySet()) {
        final String key = entry.getKey();
        final Counter value = entry.getValue();
        final String category;
        if (key.contains("-")) {
          category = key.split("-")[0];
        } else {
          category = key;
        }
        int currentCount = categoryCounter.getOrDefault(category, 0);
        categoryCounter.put(category, currentCount + value.value());
      }
    }

    @Override
    public int compare(String o1, String o2) {
      if (o1.equals(o2)) {
        return 0;
      }
      String c1 = o1;
      String c2 = o2;

      if (o1.contains("-")) {
        c1 = o1.split("-")[0];
      }
      if (o2.contains("-")) {
        c2 = o2.split("-")[0];
      }

      if (c1.equals(c2)) { // same category - sort by confusion matrix

        Counter t1 = labelCounter.get(o1);
        Counter t2 = labelCounter.get(o2);
        if (t1 == null || t2 == null) {
          if (t1 == null) {
            return 1;
          } else {
            return -1;
          }
        }
        int r1 = t1.value();
        int r2 = t2.value();
        if (r1 == r2) {
          return o1.compareTo(o2);
        }
        if (r2 > r1) {
          return 1;
        }
        return -1;
      } else { // different category - sort by category
        Integer t1 = categoryCounter.get(c1);
        Integer t2 = categoryCounter.get(c2);
        if (t1 == null || t2 == null) {
          if (t1 == null) {
            return 1;
          } else {
            return -1;
          }
        }
        if (t1.equals(t2)) {
          return o1.compareTo(o2);
        }
        if (t2 > t1) {
          return 1;
        }
        return -1;
      }
    }
  }

  /**
   * Represents a line in the confusion table.
   */
  public static class ConfusionMatrixLine {

    private final Map<String, Counter> line = new HashMap<>();
    private final String ref;
    private int total = 0;
    private int correct = 0;
    private double acc = -1;

    /**
     * Creates a new {@link ConfusionMatrixLine}
     *
     * @param ref
     *          the reference column
     */
    private ConfusionMatrixLine(String ref) {
      this.ref = ref;
    }

    /**
     * Increments the counter for the given column and updates the statistics.
     *
     * @param column
     *          the column to be incremented
     */
    private void increment(String column) {
      total++;
      if (column.equals(ref))
        correct++;
      if (!line.containsKey(column)) {
        line.put(column, new Counter());
      }
      line.get(column).increment();
    }

    /**
     * @return Retrieves the calculated accuracy of this element
     */
    public double getAccuracy() {
      // we save the accuracy because it is frequently used by the comparator
      if (StrictMath.abs(acc - 1.0d) < 0.0000000001) {
        if (total == 0)
          acc = 0.0d;
        acc = (double) correct / (double) total;
      }
      return acc;
    }

    /**
     * @param column The column to retrieve the counter value for.
     * @return Retrieves the counter value for the given {@code column}.
     */
    public int getValue(String column) {
      Counter c = line.get(column);
      if (c == null)
        return 0;
      return c.value();
    }
  }

  /**
   * Implements a simple counter
   */
  public static class Counter {
    private int c = 0;

    private void increment() {
      c++;
    }

    public int value() {
      return c;
    }
  }

  public class Stats {

    // general statistics
    private final Mean accuracy = new Mean();
    private final Mean averageSentenceLength = new Mean();
    // token statistics
    private final Map<String, Mean> tokAccuracies = new HashMap<>();
    private final Map<String, Counter> tokOccurrences = new HashMap<>();
    private final Map<String, Counter> tokErrors = new HashMap<>();
    // tag statistics
    private final Map<String, Counter> tagOccurrences = new HashMap<>();
    private final Map<String, Counter> tagErrors = new HashMap<>();
    private final Map<String, FMeasure> tagFMeasure = new HashMap<>();
    // represents a Confusion Matrix that aggregates all tokens
    private final Map<String, ConfusionMatrixLine> generalConfusionMatrix = new HashMap<>();
    // represents a set of Confusion Matrix for each token
    private final Map<String, Map<String, ConfusionMatrixLine>> tokenConfusionMatrix = new HashMap<>();
    private int minimalSentenceLength = Integer.MAX_VALUE;
    private int maximumSentenceLength = Integer.MIN_VALUE;

    public void add(String[] toks, String[] refs, String[] preds) {
      int length = toks.length;
      averageSentenceLength.add(length);

      if (minimalSentenceLength > length) {
        minimalSentenceLength = length;
      }
      if (maximumSentenceLength < length) {
        maximumSentenceLength = length;
      }

      updateTagFMeasure(refs, preds);

      for (int i = 0; i < toks.length; i++) {
        commit(toks[i], refs[i], preds[i]);
      }
    }

    public void add(int length, String ref, String pred) {

      averageSentenceLength.add(length);

      if (minimalSentenceLength > length) {
        minimalSentenceLength = length;
      }
      if (maximumSentenceLength < length) {
        maximumSentenceLength = length;
      }

      // String[] toks = reference.getSentence();
      String[] refs = { ref };
      String[] preds = { pred };

      updateTagFMeasure(refs, preds);

      commit("", ref, pred);
    }

    public void add(String[] text, String ref, String pred) {
      int length = text.length;
      this.add(length, ref, pred);
    }

    public void add(CharSequence text, String ref, String pred) {
      int length = text.length();
      this.add(length, ref, pred);
    }


    /**
     * Includes new evaluation data.
     *
     * @param tok
     *          the evaluated token
     * @param ref
     *          the reference pos tag
     * @param pred
     *          the predicted pos tag
     */
    private void commit(String tok, String ref, String pred) {
      // token stats
      if (!tokAccuracies.containsKey(tok)) {
        tokAccuracies.put(tok, new Mean());
        tokOccurrences.put(tok, new Counter());
        tokErrors.put(tok, new Counter());
      }
      tokOccurrences.get(tok).increment();

      // tag stats
      if (!tagOccurrences.containsKey(ref)) {
        tagOccurrences.put(ref, new Counter());
        tagErrors.put(ref, new Counter());
      }
      tagOccurrences.get(ref).increment();

      // updates general, token and tag error stats
      if (ref.equals(pred)) {
        tokAccuracies.get(tok).add(1);
        accuracy.add(1);
      } else {
        tokAccuracies.get(tok).add(0);
        tokErrors.get(tok).increment();
        tagErrors.get(ref).increment();
        accuracy.add(0);
      }

      // populate confusion matrixes
      if (!generalConfusionMatrix.containsKey(ref)) {
        generalConfusionMatrix.put(ref, new ConfusionMatrixLine(ref));
      }
      generalConfusionMatrix.get(ref).increment(pred);

      if (!tokenConfusionMatrix.containsKey(tok)) {
        tokenConfusionMatrix.put(tok, new HashMap<>());
      }
      if (!tokenConfusionMatrix.get(tok).containsKey(ref)) {
        tokenConfusionMatrix.get(tok).put(ref, new ConfusionMatrixLine(ref));
      }
      tokenConfusionMatrix.get(tok).get(ref).increment(pred);
    }

    private void updateTagFMeasure(String[] refs, String[] preds) {
      // create a set with all tags
      Set<String> tags = new HashSet<>(Arrays.asList(refs));
      tags.addAll(Arrays.asList(preds));

      // create samples for each tag
      for (String tag : tags) {
        List<Span> reference = new ArrayList<>();
        List<Span> prediction = new ArrayList<>();
        for (int i = 0; i < refs.length; i++) {
          if (refs[i].equals(tag)) {
            reference.add(new Span(i, i + 1));
          }
          if (preds[i].equals(tag)) {
            prediction.add(new Span(i, i + 1));
          }
        }
        if (!this.tagFMeasure.containsKey(tag)) {
          this.tagFMeasure.put(tag, new FMeasure());
        }
        // populate the F-measure
        this.tagFMeasure.get(tag).updateScores(
            reference.toArray(new Span[0]),
            prediction.toArray(new Span[0]));
      }
    }

    private double getAccuracy() {
      return accuracy.mean();
    }

    private int getNumberOfTags() {
      return this.tagOccurrences.keySet().size();
    }

    private long getNumberOfSentences() {
      return this.averageSentenceLength.count();
    }

    private double getAverageSentenceSize() {
      return this.averageSentenceLength.mean();
    }

    private int getMinSentenceSize() {
      return this.minimalSentenceLength;
    }

    private int getMaxSentenceSize() {
      return this.maximumSentenceLength;
    }

    private double getTokenAccuracy(String token) {
      return tokAccuracies.get(token).mean();
    }

    private int getTokenErrors(String token) {
      return tokErrors.get(token).value();
    }

    private int getTokenFrequency(String token) {
      return tokOccurrences.get(token).value();
    }

    private SortedSet<String> getTokensOrderedByFrequency() {
      SortedSet<String> toks = new TreeSet<>(new SimpleLabelComparator(tokOccurrences));
      toks.addAll(tokOccurrences.keySet());
      return Collections.unmodifiableSortedSet(toks);
    }

    private SortedSet<String> getTokensOrderedByNumberOfErrors() {
      SortedSet<String> toks = new TreeSet<>(new SimpleLabelComparator(tokErrors));
      toks.addAll(tokErrors.keySet());
      return toks;
    }

    private int getTagFrequency(String tag) {
      return tagOccurrences.get(tag).value();
    }

    private int getTagErrors(String tag) {
      return tagErrors.get(tag).value();
    }

    private double getTagFMeasure(String tag) {
      return tagFMeasure.get(tag).getFMeasure();
    }

    private double getTagRecall(String tag) {
      return tagFMeasure.get(tag).getRecallScore();
    }

    private double getTagPrecision(String tag) {
      return tagFMeasure.get(tag).getPrecisionScore();
    }

    private SortedSet<String> getTagsOrderedByErrors() {
      SortedSet<String> tags = new TreeSet<>(getLabelComparator(tagErrors));
      tags.addAll(tagErrors.keySet());
      return Collections.unmodifiableSortedSet(tags);
    }

    private SortedSet<String> getConfusionMatrixTagset() {
      return getConfusionMatrixTagset(generalConfusionMatrix);
    }

    private double[][] getConfusionMatrix() {
      return createConfusionMatrix(getConfusionMatrixTagset(),
          generalConfusionMatrix);
    }

    private SortedSet<String> getConfusionMatrixTagset(String token) {
      return getConfusionMatrixTagset(tokenConfusionMatrix.get(token));
    }

    private double[][] getConfusionMatrix(String token) {
      return createConfusionMatrix(getConfusionMatrixTagset(token),
          tokenConfusionMatrix.get(token));
    }

    /**
     * Creates a matrix with {@code N} lines and {@code N + 1} columns with the data from
     * confusion matrix. The last column is the accuracy.
     */
    private double[][] createConfusionMatrix(SortedSet<String> tagset,
                                             Map<String, ConfusionMatrixLine> data) {
      int size = tagset.size();
      double[][] matrix = new double[size][size + 1];
      int line = 0;
      for (String ref : tagset) {
        int column = 0;
        for (String pred : tagset) {
          matrix[line][column] = data.get(ref) != null ? data
              .get(ref).getValue(pred) : 0;
          column++;
        }
        // set accuracy
        matrix[line][column] = data.get(ref) != null ? data.get(ref).getAccuracy() : 0;
        line++;
      }

      return matrix;
    }

    private SortedSet<String> getConfusionMatrixTagset(
        Map<String, ConfusionMatrixLine> data) {
      SortedSet<String> tags = new TreeSet<>(getMatrixLabelComparator(data));
      tags.addAll(data.keySet());
      List<String> col = new LinkedList<>();
      for (String t : tags) {
        col.addAll(data.get(t).line.keySet());
      }
      tags.addAll(col);
      return Collections.unmodifiableSortedSet(tags);
    }
  }
}
