Summary

keras-yolo3-master/utils/bbox.py

@@ -0,0 +1,89 @@
+import numpy as np
+import os
+import cv2
+from .colors import get_color
+
+class BoundBox:
+    def __init__(self, xmin, ymin, xmax, ymax, c = None, classes = None):
+        self.xmin = xmin
+        self.ymin = ymin
+        self.xmax = xmax
+        self.ymax = ymax
+        
+        self.c       = c
+        self.classes = classes
+
+        self.label = -1
+        self.score = -1
+
+    def get_label(self):
+        if self.label == -1:
+            self.label = np.argmax(self.classes)
+        
+        return self.label
+    
+    def get_score(self):
+        if self.score == -1:
+            self.score = self.classes[self.get_label()]
+            
+        return self.score      
+
+def _interval_overlap(interval_a, interval_b):
+    x1, x2 = interval_a
+    x3, x4 = interval_b
+
+    if x3 < x1:
+        if x4 < x1:
+            return 0
+        else:
+            return min(x2,x4) - x1
+    else:
+        if x2 < x3:
+             return 0
+        else:
+            return min(x2,x4) - x3    
+
+def bbox_iou(box1, box2):
+    intersect_w = _interval_overlap([box1.xmin, box1.xmax], [box2.xmin, box2.xmax])
+    intersect_h = _interval_overlap([box1.ymin, box1.ymax], [box2.ymin, box2.ymax])  
+    
+    intersect = intersect_w * intersect_h
+
+    w1, h1 = box1.xmax-box1.xmin, box1.ymax-box1.ymin
+    w2, h2 = box2.xmax-box2.xmin, box2.ymax-box2.ymin
+    
+    union = w1*h1 + w2*h2 - intersect
+    
+    return float(intersect) / union
+
+def draw_boxes(image, boxes, labels, obj_thresh, quiet=True):
+    for box in boxes:
+        label_str = ''
+        label = -1
+        
+        for i in range(len(labels)):
+            if box.classes[i] > obj_thresh:
+                if label_str != '': label_str += ', '
+                label_str += (labels[i] + ' ' + str(round(box.get_score()*100,0)) + '%')
+                label = i
+            if not quiet: print(label_str)
+                
+        if label >= 0:
+            text_size = cv2.getTextSize(label_str, cv2.FONT_HERSHEY_SIMPLEX, 1.1e-4 * image.shape[0], 2)
+            width, height = text_size[0][0], text_size[0][1]
+            region = np.array([[box.xmin-3,        box.ymin], 
+                               [box.xmin-3,        box.ymin-height-16], 
+                               [box.xmin+width+6, box.ymin-height-16], 
+                               [box.xmin+width+6, box.ymin]], dtype='int32')  
+
+            cv2.rectangle(img=image, pt1=(box.xmin,box.ymin), pt2=(box.xmax,box.ymax), color=get_color(label), thickness=1)
+            cv2.fillPoly(img=image, pts=[region], color=get_color(label))
+            cv2.putText(img=image, 
+                        text=label_str, 
+                        org=(box.xmin+6, box.ymin - 6), 
+                        fontFace=cv2.FONT_HERSHEY_SIMPLEX, 
+                        fontScale=0.7e-3 * image.shape[0], 
+                        color=(0,0,0), 
+                        thickness=2)
+        
+    return image          

keras-yolo3-master/utils/colors.py

@@ -0,0 +1,96 @@
+def get_color(label):
+    """ Return a color from a set of predefined colors. Contains 80 colors in total.
+    code originally from https://github.com/fizyr/keras-retinanet/
+    Args
+        label: The label to get the color for.
+    Returns
+        A list of three values representing a RGB color.
+    """
+    if label < len(colors):
+        return colors[label]
+    else:
+        print('Label {} has no color, returning default.'.format(label))
+        return (0, 255, 0)
+
+colors = [
+    [31  , 0   , 255] ,
+    [0   , 159 , 255] ,
+    [255 , 95  , 0]   ,
+    [255 , 19  , 0]   ,
+    [255 , 0   , 0]   ,
+    [255 , 38  , 0]   ,
+    [0   , 255 , 25]  ,
+    [255 , 0   , 133] ,
+    [255 , 172 , 0]   ,
+    [108 , 0   , 255] ,
+    [0   , 82  , 255] ,
+    [0   , 255 , 6]   ,
+    [255 , 0   , 152] ,
+    [223 , 0   , 255] ,
+    [12  , 0   , 255] ,
+    [0   , 255 , 178] ,
+    [108 , 255 , 0]   ,
+    [184 , 0   , 255] ,
+    [255 , 0   , 76]  ,
+    [146 , 255 , 0]   ,
+    [51  , 0   , 255] ,
+    [0   , 197 , 255] ,
+    [255 , 248 , 0]   ,
+    [255 , 0   , 19]  ,
+    [255 , 0   , 38]  ,
+    [89  , 255 , 0]   ,
+    [127 , 255 , 0]   ,
+    [255 , 153 , 0]   ,
+    [0   , 255 , 255] ,
+    [0   , 255 , 216] ,
+    [0   , 255 , 121] ,
+    [255 , 0   , 248] ,
+    [70  , 0   , 255] ,
+    [0   , 255 , 159] ,
+    [0   , 216 , 255] ,
+    [0   , 6   , 255] ,
+    [0   , 63  , 255] ,
+    [31  , 255 , 0]   ,
+    [255 , 57  , 0]   ,
+    [255 , 0   , 210] ,
+    [0   , 255 , 102] ,
+    [242 , 255 , 0]   ,
+    [255 , 191 , 0]   ,
+    [0   , 255 , 63]  ,
+    [255 , 0   , 95]  ,
+    [146 , 0   , 255] ,
+    [184 , 255 , 0]   ,
+    [255 , 114 , 0]   ,
+    [0   , 255 , 235] ,
+    [255 , 229 , 0]   ,
+    [0   , 178 , 255] ,
+    [255 , 0   , 114] ,
+    [255 , 0   , 57]  ,
+    [0   , 140 , 255] ,
+    [0   , 121 , 255] ,
+    [12  , 255 , 0]   ,
+    [255 , 210 , 0]   ,
+    [0   , 255 , 44]  ,
+    [165 , 255 , 0]   ,
+    [0   , 25  , 255] ,
+    [0   , 255 , 140] ,
+    [0   , 101 , 255] ,
+    [0   , 255 , 82]  ,
+    [223 , 255 , 0]   ,
+    [242 , 0   , 255] ,
+    [89  , 0   , 255] ,
+    [165 , 0   , 255] ,
+    [70  , 255 , 0]   ,
+    [255 , 0   , 172] ,
+    [255 , 76  , 0]   ,
+    [203 , 255 , 0]   ,
+    [204 , 0   , 255] ,
+    [255 , 0   , 229] ,
+    [255 , 133 , 0]   ,
+    [127 , 0   , 255] ,
+    [0   , 235 , 255] ,
+    [0   , 255 , 197] ,
+    [255 , 0   , 191] ,
+    [0   , 44  , 255] ,
+    [50  , 255 , 0]
+]

keras-yolo3-master/utils/image.py

@@ -0,0 +1,86 @@
+import cv2
+import numpy as np
+import copy
+
+def _rand_scale(scale):
+    scale = np.random.uniform(1, scale)
+    return scale if (np.random.randint(2) == 0) else 1./scale;
+
+def _constrain(min_v, max_v, value):
+    if value < min_v: return min_v
+    if value > max_v: return max_v
+    return value 
+
+def random_flip(image, flip):
+    if flip == 1: return cv2.flip(image, 1)
+    return image
+
+def correct_bounding_boxes(boxes, new_w, new_h, net_w, net_h, dx, dy, flip, image_w, image_h):
+    boxes = copy.deepcopy(boxes)
+
+    # randomize boxes' order
+    np.random.shuffle(boxes)
+
+    # correct sizes and positions
+    sx, sy = float(new_w)/image_w, float(new_h)/image_h
+    zero_boxes = []
+
+    for i in range(len(boxes)):
+        boxes[i]['xmin'] = int(_constrain(0, net_w, boxes[i]['xmin']*sx + dx))
+        boxes[i]['xmax'] = int(_constrain(0, net_w, boxes[i]['xmax']*sx + dx))
+        boxes[i]['ymin'] = int(_constrain(0, net_h, boxes[i]['ymin']*sy + dy))
+        boxes[i]['ymax'] = int(_constrain(0, net_h, boxes[i]['ymax']*sy + dy))
+
+        if boxes[i]['xmax'] <= boxes[i]['xmin'] or boxes[i]['ymax'] <= boxes[i]['ymin']:
+            zero_boxes += [i]
+            continue
+
+        if flip == 1:
+            swap = boxes[i]['xmin'];
+            boxes[i]['xmin'] = net_w - boxes[i]['xmax']
+            boxes[i]['xmax'] = net_w - swap
+
+    boxes = [boxes[i] for i in range(len(boxes)) if i not in zero_boxes]
+
+    return boxes
+
+def random_distort_image(image, hue=18, saturation=1.5, exposure=1.5):
+    # determine scale factors
+    dhue = np.random.uniform(-hue, hue)
+    dsat = _rand_scale(saturation);
+    dexp = _rand_scale(exposure);     
+
+    # convert RGB space to HSV space
+    image = cv2.cvtColor(image, cv2.COLOR_RGB2HSV).astype('float')
+    
+    # change satuation and exposure
+    image[:,:,1] *= dsat
+    image[:,:,2] *= dexp
+    
+    # change hue
+    image[:,:,0] += dhue
+    image[:,:,0] -= (image[:,:,0] > 180)*180
+    image[:,:,0] += (image[:,:,0] < 0)  *180
+    
+    # convert back to RGB from HSV
+    return cv2.cvtColor(image.astype('uint8'), cv2.COLOR_HSV2RGB)
+
+def apply_random_scale_and_crop(image, new_w, new_h, net_w, net_h, dx, dy):
+    im_sized = cv2.resize(image, (new_w, new_h))
+    
+    if dx > 0: 
+        im_sized = np.pad(im_sized, ((0,0), (dx,0), (0,0)), mode='constant', constant_values=127)
+    else:
+        im_sized = im_sized[:,-dx:,:]
+    if (new_w + dx) < net_w:
+        im_sized = np.pad(im_sized, ((0,0), (0, net_w - (new_w+dx)), (0,0)), mode='constant', constant_values=127)
+               
+    if dy > 0: 
+        im_sized = np.pad(im_sized, ((dy,0), (0,0), (0,0)), mode='constant', constant_values=127)
+    else:
+        im_sized = im_sized[-dy:,:,:]
+        
+    if (new_h + dy) < net_h:
+        im_sized = np.pad(im_sized, ((0, net_h - (new_h+dy)), (0,0), (0,0)), mode='constant', constant_values=127)
+        
+    return im_sized[:net_h, :net_w,:]     

keras-yolo3-master/utils/multi_gpu_model.py

@@ -0,0 +1,62 @@
+from keras.layers import Lambda, concatenate
+from keras.models import Model
+import tensorflow as tf
+
+def multi_gpu_model(model, gpus):
+    if isinstance(gpus, (list, tuple)):
+        num_gpus = len(gpus)
+        target_gpu_ids = gpus
+    else:
+        num_gpus = gpus
+        target_gpu_ids = range(num_gpus)
+
+    def get_slice(data, i, parts):
+        shape = tf.shape(data)
+        batch_size = shape[:1]
+        input_shape = shape[1:]
+        step = batch_size // parts
+        if i == num_gpus - 1:
+            size = batch_size - step * i
+        else:
+            size = step
+        size = tf.concat([size, input_shape], axis=0)
+        stride = tf.concat([step, input_shape * 0], axis=0)
+        start = stride * i
+        return tf.slice(data, start, size)
+
+    all_outputs = []
+    for i in range(len(model.outputs)):
+        all_outputs.append([])
+
+    # Place a copy of the model on each GPU,
+    # each getting a slice of the inputs.
+    for i, gpu_id in enumerate(target_gpu_ids):
+        with tf.device('/gpu:%d' % gpu_id):
+            with tf.name_scope('replica_%d' % gpu_id):
+                inputs = []
+                # Retrieve a slice of the input.
+                for x in model.inputs:
+                    input_shape = tuple(x.get_shape().as_list())[1:]
+                    slice_i = Lambda(get_slice,
+                                                     output_shape=input_shape,
+                                                     arguments={'i': i,
+                                                                            'parts': num_gpus})(x)
+                    inputs.append(slice_i)
+
+                # Apply model on slice
+                # (creating a model replica on the target device).
+                outputs = model(inputs)
+                if not isinstance(outputs, list):
+                    outputs = [outputs]
+
+                # Save the outputs for merging back together later.
+                for o in range(len(outputs)):
+                    all_outputs[o].append(outputs[o])
+
+    # Merge outputs on CPU.
+    with tf.device('/cpu:0'):
+        merged = []
+        for name, outputs in zip(model.output_names, all_outputs):
+            merged.append(concatenate(outputs,
+                                                                axis=0, name=name))
+        return Model(model.inputs, merged)

keras-yolo3-master/utils/utils.py

@@ -0,0 +1,323 @@
+import cv2
+import numpy as np
+import os
+from .bbox import BoundBox, bbox_iou
+from scipy.special import expit
+
+def _sigmoid(x):
+    return expit(x)
+
+def makedirs(path):
+    try:
+        os.makedirs(path)
+    except OSError:
+        if not os.path.isdir(path):
+            raise
+
+def evaluate(model,
+             generator,
+             iou_threshold=0.5,
+             obj_thresh=0.5,
+             nms_thresh=0.45,
+             net_h=416,
+             net_w=416,
+             save_path=None):
+    """ Evaluate a given dataset using a given model.
+    code originally from https://github.com/fizyr/keras-retinanet
+
+    # Arguments
+        model           : The model to evaluate.
+        generator       : The generator that represents the dataset to evaluate.
+        iou_threshold   : The threshold used to consider when a detection is positive or negative.
+        obj_thresh      : The threshold used to distinguish between object and non-object
+        nms_thresh      : The threshold used to determine whether two detections are duplicates
+        net_h           : The height of the input image to the model, higher value results in better accuracy
+        net_w           : The width of the input image to the model
+        save_path       : The path to save images with visualized detections to.
+    # Returns
+        A dict mapping class names to mAP scores.
+    """
+    # gather all detections and annotations
+    all_detections     = [[None for i in range(generator.num_classes())] for j in range(generator.size())]
+    all_annotations    = [[None for i in range(generator.num_classes())] for j in range(generator.size())]
+
+    for i in range(generator.size()):
+        raw_image = [generator.load_image(i)]
+
+        # make the boxes and the labels
+        pred_boxes = get_yolo_boxes(model, raw_image, net_h, net_w, generator.get_anchors(), obj_thresh, nms_thresh)[0]
+
+        score = np.array([box.get_score() for box in pred_boxes])
+        pred_labels = np.array([box.label for box in pred_boxes])
+
+        if len(pred_boxes) > 0:
+            pred_boxes = np.array([[box.xmin, box.ymin, box.xmax, box.ymax, box.get_score()] for box in pred_boxes])
+        else:
+            pred_boxes = np.array([[]])
+
+        # sort the boxes and the labels according to scores
+        score_sort = np.argsort(-score)
+        pred_labels = pred_labels[score_sort]
+        pred_boxes  = pred_boxes[score_sort]
+
+        # copy detections to all_detections
+        for label in range(generator.num_classes()):
+            all_detections[i][label] = pred_boxes[pred_labels == label, :]
+
+        annotations = generator.load_annotation(i)
+
+        # copy detections to all_annotations
+        for label in range(generator.num_classes()):
+            all_annotations[i][label] = annotations[annotations[:, 4] == label, :4].copy()
+
+    # compute mAP by comparing all detections and all annotations
+    average_precisions = {}
+
+    for label in range(generator.num_classes()):
+        false_positives = np.zeros((0,))
+        true_positives  = np.zeros((0,))
+        scores          = np.zeros((0,))
+        num_annotations = 0.0
+
+        for i in range(generator.size()):
+            detections           = all_detections[i][label]
+            annotations          = all_annotations[i][label]
+            num_annotations     += annotations.shape[0]
+            detected_annotations = []
+
+            for d in detections:
+                scores = np.append(scores, d[4])
+
+                if annotations.shape[0] == 0:
+                    false_positives = np.append(false_positives, 1)
+                    true_positives  = np.append(true_positives, 0)
+                    continue
+
+                overlaps            = compute_overlap(np.expand_dims(d, axis=0), annotations)
+                assigned_annotation = np.argmax(overlaps, axis=1)
+                max_overlap         = overlaps[0, assigned_annotation]
+
+                if max_overlap >= iou_threshold and assigned_annotation not in detected_annotations:
+                    false_positives = np.append(false_positives, 0)
+                    true_positives  = np.append(true_positives, 1)
+                    detected_annotations.append(assigned_annotation)
+                else:
+                    false_positives = np.append(false_positives, 1)
+                    true_positives  = np.append(true_positives, 0)
+
+        # no annotations -> AP for this class is 0 (is this correct?)
+        if num_annotations == 0:
+            average_precisions[label] = 0
+            continue
+
+        # sort by score
+        indices         = np.argsort(-scores)
+        false_positives = false_positives[indices]
+        true_positives  = true_positives[indices]
+
+        # compute false positives and true positives
+        false_positives = np.cumsum(false_positives)
+        true_positives  = np.cumsum(true_positives)
+
+        # compute recall and precision
+        recall    = true_positives / num_annotations
+        precision = true_positives / np.maximum(true_positives + false_positives, np.finfo(np.float64).eps)
+
+        # compute average precision
+        average_precision  = compute_ap(recall, precision)
+        average_precisions[label] = average_precision,num_annotations
+
+    return average_precisions
+
+def correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w):
+    if (float(net_w)/image_w) < (float(net_h)/image_h):
+        new_w = net_w
+        new_h = (image_h*net_w)/image_w
+    else:
+        new_h = net_w
+        new_w = (image_w*net_h)/image_h
+
+    for i in range(len(boxes)):
+        x_offset, x_scale = (net_w - new_w)/2./net_w, float(new_w)/net_w
+        y_offset, y_scale = (net_h - new_h)/2./net_h, float(new_h)/net_h
+
+        boxes[i].xmin = int((boxes[i].xmin - x_offset) / x_scale * image_w)
+        boxes[i].xmax = int((boxes[i].xmax - x_offset) / x_scale * image_w)
+        boxes[i].ymin = int((boxes[i].ymin - y_offset) / y_scale * image_h)
+        boxes[i].ymax = int((boxes[i].ymax - y_offset) / y_scale * image_h)
+
+def do_nms(boxes, nms_thresh):
+    if len(boxes) > 0:
+        nb_class = len(boxes[0].classes)
+    else:
+        return
+
+    for c in range(nb_class):
+        sorted_indices = np.argsort([-box.classes[c] for box in boxes])
+
+        for i in range(len(sorted_indices)):
+            index_i = sorted_indices[i]
+
+            if boxes[index_i].classes[c] == 0: continue
+
+            for j in range(i+1, len(sorted_indices)):
+                index_j = sorted_indices[j]
+
+                if bbox_iou(boxes[index_i], boxes[index_j]) >= nms_thresh:
+                    boxes[index_j].classes[c] = 0
+
+def decode_netout(netout, anchors, obj_thresh, net_h, net_w):
+    grid_h, grid_w = netout.shape[:2]
+    nb_box = 3
+    netout = netout.reshape((grid_h, grid_w, nb_box, -1))
+    nb_class = netout.shape[-1] - 5
+
+    boxes = []
+
+    netout[..., :2]  = _sigmoid(netout[..., :2])
+    netout[..., 4]   = _sigmoid(netout[..., 4])
+    netout[..., 5:]  = netout[..., 4][..., np.newaxis] * _softmax(netout[..., 5:])
+    netout[..., 5:] *= netout[..., 5:] > obj_thresh
+
+    for i in range(grid_h*grid_w):
+        row = i // grid_w
+        col = i % grid_w
+
+        for b in range(nb_box):
+            # 4th element is objectness score
+            objectness = netout[row, col, b, 4]
+
+            if(objectness <= obj_thresh): continue
+
+            # first 4 elements are x, y, w, and h
+            x, y, w, h = netout[row,col,b,:4]
+
+            x = (col + x) / grid_w # center position, unit: image width
+            y = (row + y) / grid_h # center position, unit: image height
+            w = anchors[2 * b + 0] * np.exp(w) / net_w # unit: image width
+            h = anchors[2 * b + 1] * np.exp(h) / net_h # unit: image height
+
+            # last elements are class probabilities
+            classes = netout[row,col,b,5:]
+
+            box = BoundBox(x-w/2, y-h/2, x+w/2, y+h/2, objectness, classes)
+
+            boxes.append(box)
+
+    return boxes
+
+def preprocess_input(image, net_h, net_w):
+    new_h, new_w, _ = image.shape
+
+    # determine the new size of the image
+    if (float(net_w)/new_w) < (float(net_h)/new_h):
+        new_h = (new_h * net_w)//new_w
+        new_w = net_w
+    else:
+        new_w = (new_w * net_h)//new_h
+        new_h = net_h
+
+    # resize the image to the new size
+    resized = cv2.resize(image[:,:,::-1]/255., (new_w, new_h))
+
+    # embed the image into the standard letter box
+    new_image = np.ones((net_h, net_w, 3)) * 0.5
+    new_image[(net_h-new_h)//2:(net_h+new_h)//2, (net_w-new_w)//2:(net_w+new_w)//2, :] = resized
+    new_image = np.expand_dims(new_image, 0)
+
+    return new_image
+
+def normalize(image):
+    return image/255.
+
+def get_yolo_boxes(model, images, net_h, net_w, anchors, obj_thresh, nms_thresh):
+    image_h, image_w, _ = images[0].shape
+    nb_images           = len(images)
+    batch_input         = np.zeros((nb_images, net_h, net_w, 3))
+
+    # preprocess the input
+    for i in range(nb_images):
+        batch_input[i] = preprocess_input(images[i], net_h, net_w)
+
+    # run the prediction
+    batch_output = model.predict_on_batch(batch_input)
+    batch_boxes  = [None]*nb_images
+
+    for i in range(nb_images):
+        yolos = [batch_output[0][i], batch_output[1][i], batch_output[2][i]]
+        boxes = []
+
+        # decode the output of the network
+        for j in range(len(yolos)):
+            yolo_anchors = anchors[(2-j)*6:(3-j)*6] # config['model']['anchors']
+            boxes += decode_netout(yolos[j], yolo_anchors, obj_thresh, net_h, net_w)
+
+        # correct the sizes of the bounding boxes
+        correct_yolo_boxes(boxes, image_h, image_w, net_h, net_w)
+
+        # suppress non-maximal boxes
+        do_nms(boxes, nms_thresh)
+
+        batch_boxes[i] = boxes
+
+    return batch_boxes
+
+def compute_overlap(a, b):
+    """
+    Code originally from https://github.com/rbgirshick/py-faster-rcnn.
+    Parameters
+    ----------
+    a: (N, 4) ndarray of float
+    b: (K, 4) ndarray of float
+    Returns
+    -------
+    overlaps: (N, K) ndarray of overlap between boxes and query_boxes
+    """
+    area = (b[:, 2] - b[:, 0]) * (b[:, 3] - b[:, 1])
+
+    iw = np.minimum(np.expand_dims(a[:, 2], axis=1), b[:, 2]) - np.maximum(np.expand_dims(a[:, 0], 1), b[:, 0])
+    ih = np.minimum(np.expand_dims(a[:, 3], axis=1), b[:, 3]) - np.maximum(np.expand_dims(a[:, 1], 1), b[:, 1])
+
+    iw = np.maximum(iw, 0)
+    ih = np.maximum(ih, 0)
+
+    ua = np.expand_dims((a[:, 2] - a[:, 0]) * (a[:, 3] - a[:, 1]), axis=1) + area - iw * ih
+
+    ua = np.maximum(ua, np.finfo(float).eps)
+
+    intersection = iw * ih
+
+    return intersection / ua
+
+def compute_ap(recall, precision):
+    """ Compute the average precision, given the recall and precision curves.
+    Code originally from https://github.com/rbgirshick/py-faster-rcnn.
+
+    # Arguments
+        recall:    The recall curve (list).
+        precision: The precision curve (list).
+    # Returns
+        The average precision as computed in py-faster-rcnn.
+    """
+    # correct AP calculation
+    # first append sentinel values at the end
+    mrec = np.concatenate(([0.], recall, [1.]))
+    mpre = np.concatenate(([0.], precision, [0.]))
+
+    # compute the precision envelope
+    for i in range(mpre.size - 1, 0, -1):
+        mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i])
+
+    # to calculate area under PR curve, look for points
+    # where X axis (recall) changes value
+    i = np.where(mrec[1:] != mrec[:-1])[0]
+
+    # and sum (\Delta recall) * prec
+    ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1])
+    return ap
+
+def _softmax(x, axis=-1):
+    x = x - np.amax(x, axis, keepdims=True)
+    e_x = np.exp(x)
+
+    return e_x / e_x.sum(axis, keepdims=True)