content: # Copyright 2017 The TensorFlow Authors. All Rights Reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  # ==============================================================================
  
  """Functions for computing metrics like precision, recall, CorLoc and etc."""
  from __future__ import division
  
  import numpy as np
  
  
  def compute_precision_recall(scores, labels, num_gt):
    """Compute precision and recall.
  
    Args:
      scores: A float numpy array representing detection score
      labels: A boolean numpy array representing true/false positive labels
      num_gt: Number of ground truth instances
  
    Raises:
      ValueError: if the input is not of the correct format
  
    Returns:
      precision: Fraction of positive instances over detected ones. This value is
        None if no ground truth labels are present.
      recall: Fraction of detected positive instance over all positive instances.
        This value is None if no ground truth labels are present.
  
    """
    if not isinstance(
        labels, np.ndarray) or labels.dtype != np.bool or len(labels.shape) != 1:
      raise ValueError("labels must be single dimension bool numpy array")
  
    if not isinstance(
        scores, np.ndarray) or len(scores.shape) != 1:
      raise ValueError("scores must be single dimension numpy array")
  
    if num_gt < np.sum(labels):
      raise ValueError("Number of true positives must be smaller than num_gt.")
  
    if len(scores) != len(labels):
      raise ValueError("scores and labels must be of the same size.")
  
    if num_gt == 0:
      return None, None
  
    sorted_indices = np.argsort(scores)
    sorted_indices = sorted_indices[::-1]
    labels = labels.astype(int)
    true_positive_labels = labels[sorted_indices]
    false_positive_labels = 1 - true_positive_labels
    cum_true_positives = np.cumsum(true_positive_labels)
    cum_false_positives = np.cumsum(false_positive_labels)
    precision = cum_true_positives.astype(float) / (
        cum_true_positives + cum_false_positives)
    recall = cum_true_positives.astype(float) / num_gt
    return precision, recall
  
  
  def compute_average_precision(precision, recall):
    """Compute Average Precision according to the definition in VOCdevkit.
  
    Precision is modified to ensure that it does not decrease as recall
    decrease.
  
    Args:
      precision: A float [N, 1] numpy array of precisions
      recall: A float [N, 1] numpy array of recalls
  
    Raises:
      ValueError: if the input is not of the correct format
  
    Returns:
      average_precison: The area under the precision recall curve. NaN if
        precision and recall are None.
  
    """
    if precision is None:
      if recall is not None:
        raise ValueError("If precision is None, recall must also be None")
      return np.NAN
  
    if not isinstance(precision, np.ndarray) or not isinstance(recall,
                                                               np.ndarray):
      raise ValueError("precision and recall must be numpy array")
    if precision.dtype != np.float or recall.dtype != np.float:
      raise ValueError("input must be float numpy array.")
    if len(precision) != len(recall):
      raise ValueError("precision and recall must be of the same size.")
    if not precision.size:
      return 0.0
    if np.amin(precision) < 0 or np.amax(precision) > 1:
      raise ValueError("Precision must be in the range of [0, 1].")
    if np.amin(recall) < 0 or np.amax(recall) > 1:
      raise ValueError("recall must be in the range of [0, 1].")
    if not all(recall[i] <= recall[i + 1] for i in range(len(recall) - 1)):
      raise ValueError("recall must be a non-decreasing array")
  
    recall = np.concatenate([[0], recall, [1]])
    precision = np.concatenate([[0], precision, [0]])
  
    # Preprocess precision to be a non-decreasing array
    for i in range(len(precision) - 2, -1, -1):
      precision[i] = np.maximum(precision[i], precision[i + 1])
  
    indices = np.where(recall[1:] != recall[:-1])[0] + 1
    average_precision = np.sum(
        (recall[indices] - recall[indices - 1]) * precision[indices])
    return average_precision
  
  
  def compute_cor_loc(num_gt_imgs_per_class,
                      num_images_correctly_detected_per_class):
    """Compute CorLoc according to the definition in the following paper.
  
    https://www.robots.ox.ac.uk/~vgg/rg/papers/deselaers-eccv10.pdf
  
    Returns nans if there are no ground truth images for a class.
  
    Args:
      num_gt_imgs_per_class: 1D array, representing number of images containing
          at least one object instance of a particular class
      num_images_correctly_detected_per_class: 1D array, representing number of
          images that are correctly detected at least one object instance of a
          particular class
  
    Returns:
      corloc_per_class: A float numpy array represents the corloc score of each
        class
    """
    # Divide by zero expected for classes with no gt examples.
    with np.errstate(divide="ignore", invalid="ignore"):
      return np.where(
          num_gt_imgs_per_class == 0, np.nan,
          num_images_correctly_detected_per_class / num_gt_imgs_per_class)
  