袋熊的树洞

import numpy as np

>>> arr = np.arange(10)
>>> arr
array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
>>> new_arr = arr[5:8]
>>> new_arr
array([5, 6, 7])

>>> new_arr[:] = 12
>>> new_arr
array([12, 12, 12])
>>> arr
array([ 0,  1,  2,  3,  4, 12, 12, 12,  8,  9])

>>> new_arr = arr[5:8].copy()

>>> A = np.random.randn(2, 3)
>>> A
array([[-0.40797393,  0.24059956, -1.57582642],
       [ 0.31626161, -0.07033558,  0.58346107]])
>>> A.mean(axis=0)
array([-0.04585616,  0.08513199, -0.49618267])
>>> A.mean(axis=1)
array([-0.58106693,  0.27646237])

>>> bools = np.array([False, True, True, False, False])
>>> bools.sum()
2
>>> np.sum(bools)
2

>>> bools.mean()
0.4
>>> np.mean(bools)
0.4

{'a': 1, 'b': True}
{'a': 1, 'b': False}
{'a': 2, 'b': True}
{'a': 2, 'b': False}

$ tree test
test
├── 1.txt
├── 2.txt
└── test2
    ├── 3.txt
    └── 4.txt

/tmp/test/1.txt
/tmp/test/2.txt
/tmp/test/test2
/tmp/test/test2/3.txt
/tmp/test/test2/4.txt

/tmp/test/test2/3.txt
/tmp/test/test2/4.txt
/tmp/test/1.txt
/tmp/test/2.txt
/tmp/test/test2

'a' is iterable
'b' is not iterable

class VGG(nn.Module):
	def __init__(self, features, num_classes=1000, init_weights=True):
        super(VGG, self).__init__()
        self.features = features
        self.classifier = nn.Sequential(
            nn.Linear(512 * 7 * 7, 4096),
            nn.ReLU(True),
            nn.Dropout(),
            nn.Linear(4096, 4096),
            nn.ReLU(True),
            nn.Dropout(),
            nn.Linear(4096, num_classes),
        )
        if init_weights:
            self._initialize_weights()

self.classifier = nn.Sequential(
    nn.Linear(512 * 1 * 1, 512),
    nn.ReLU(True),
    nn.Dropout(),
    nn.Linear(512, 512),
    nn.ReLU(True),
    nn.Dropout(),
    nn.Linear(512, num_classes),
)

$ brew install hdf5 --with-mpi

$ pip3 install mpi4py

$ pip3 install --no-cache-dir mpi4py

$ export CC=mpicc
$ export HDF5_MPI="ON"
$ pip3 install --no-binary=h5py h5py

$ pip3 install --no-cache-dir --no-binary=h5py h5py

$ ls data | sort -n
1.png
2.png
3.png
4.png
5.png
6.png
7.png
8.png
9.png
10.png

$ cat labels.csv
id,label
1,frog
2,truck
3,truck
4,deer
5,automobile
6,automobile
7,bird
8,horse
9,ship
10,cat

import numpy as np
import pandas as pd
from skimage import io
import matplotlib.pyplot as plt
import os
import lmdb
import caffe


def make_datum(image, label, channels, height, width):
    datum = caffe.proto.caffe_pb2.Datum()
    datum.channels = channels
    datum.label = int(label)
    datum.height = height
    datum.width = width
    datum.data = image.tobytes()

    return datum


# data path and lmdb path
dataset_path = './data'
label_file = 'labels.csv'
lmdb_path = 'cifar10_lmdb'

# labels mapping
labels_mapping = {'airplane': 0, 'automobile': 1,
                  'bird': 2, 'cat': 3, 'deer': 4, 'dog': 5,
                  'frog': 6, 'horse': 7, 'ship': 8, 'truck': 9}
classes = {}
for key in labels_mapping:
    classes[labels_mapping[key]] = key

# load data
df = pd.read_csv(label_file)
df['label'] = df['label'].map(labels_mapping)
images = list(df.id)
labels = list(df.label)

# write data to LMDB
map_size = 1e6
batch_size = 4

count = 0
lmdb_env = lmdb.open(lmdb_path, map_size=map_size)
lmdb_txn = lmdb_env.begin(write=True)

for image_id, label in zip(images, labels):
    count = count + 1
    image_file = os.path.join(dataset_path, str(image_id) + '.png')
    image = io.imread(image_file)
    height, width, channels = image.shape
    datum = make_datum(image, label, channels, height, width)
    str_id = '{:08}'.format(count)
    lmdb_txn.put(str_id, datum.SerializeToString())

    if count % batch_size == 0:
        lmdb_txn.commit()
        lmdb_txn = lmdb_env.begin(write=True)

lmdb_txn.commit()
lmdb_env.close()

lmdb_env = lmdb.open(lmdb_path)
lmdb_txn = lmdb_env.begin()
lmdb_cursor = lmdb_txn.cursor()
datum = caffe.proto.caffe_pb2.Datum()

count = 0
for key, value in lmdb_cursor:
    datum.ParseFromString(value)
    label = datum.label
    height = datum.height
    width = datum.width
    channels = datum.channels
    data = datum.data
    count = count + 1

    if count == 2:
        image = np.frombuffer(data, dtype=np.uint8)
        image = np.reshape(image, (height, width, channels))
        print('Label: {}, Class: {}'.format(label, classes[label]))
        plt.imshow(image)
        break

print('Number of items: {}'.format(count))

lmdb_env.close()

Time simple: 41.33699381200131
Time direct: 39.005692337988876

import numpy as np


def py_cpu_nms(dets, thresh):
    """Pure Python NMS baseline."""
    x1 = dets[:, 0]
    y1 = dets[:, 1]
    x2 = dets[:, 2]
    y2 = dets[:, 3]
    scores = dets[:, 4]

    # 每个框 (bounding box) 的面积
    areas = (x2 - x1 + 1) * (y2 - y1 + 1)
    # 根据得分 (score) 的大小进行降序排序
    order = scores.argsort()[::-1]

    keep = []
    while order.size > 0:
        # 保留剩余框中得分最高的那个
        i = order[0]
        keep.append(i)

        # 计算相交区域位置，左上以及右下
        xx1 = np.maximum(x1[i], x1[order[1:]])
        yy1 = np.maximum(y1[i], y1[order[1:]])
        xx2 = np.minimum(x2[i], x2[order[1:]])
        yy2 = np.minimum(y2[i], y2[order[1:]])

        # 计算相交区域面积
        w = np.maximum(0.0, xx2 - xx1 + 1)
        h = np.maximum(0.0, yy2 - yy1 + 1)

        # 计算IoU，即 重叠面积 / (框1面积 + 框2面积 - 重叠面积)
        inter = w * h
        ovr = inter / (areas[i] + areas[order[1:]] - inter)

        # 保留IoU小于阈值的框
        inds = np.where(ovr <= thresh)[0]

        # 因为ovr数组的长度比order长度小1，所以这里要将所有下标后移一位
        order = order[inds + 1]

    return keep

$ ls -l train/ | head
total 400000
-rw-r--r--@ 1 luowanqian  staff  2455 10 19  2013 1.png
-rw-r--r--@ 1 luowanqian  staff  2101 10 19  2013 10.png
-rw-r--r--@ 1 luowanqian  staff  2466 10 19  2013 100.png
-rw-r--r--@ 1 luowanqian  staff  2171 10 19  2013 1000.png
-rw-r--r--@ 1 luowanqian  staff  2031 10 19  2013 10000.png
-rw-r--r--@ 1 luowanqian  staff  2051 10 19  2013 10001.png
-rw-r--r--@ 1 luowanqian  staff  2317 10 19  2013 10002.png
-rw-r--r--@ 1 luowanqian  staff  2511 10 19  2013 10003.png
-rw-r--r--@ 1 luowanqian  staff  2460 10 19  2013 10004.png

$ ls -l train/ | grep ".png" | tr -s ' ' | cut -d ' ' -f 9 | sort -n > list.txt
$ head list.txt
1.png
2.png
3.png
4.png
5.png
6.png
7.png
8.png
9.png
10.png

$ ls train/ | sort -n > list.txt
$ head list.txt
1.png
2.png
3.png
4.png
5.png
6.png
7.png
8.png
9.png
10.png

$ rsync -P ubuntu/ubuntu-16.04.4-desktop-amd64.iso lab217server:/home/luowanqian/Downloads/

y_true = [0, 1, 1, 0, 1, 0, 1, 1, 1, 0]
y_score = [0.505, 0.6, 0.8, 0.52, 0.55, 0.53, 0.54, 0.9, 0.51, 0.7]

y_true = [1, 1, 0, 1, 1, 1, 0, 0, 1, 0]
y_score = [0.9, 0.8, 0.7, 0.6, 0.55, 0.54, 0.53, 0.52, 0.51, 0.505]

y_true = [1, 1, 0, 1, 1, 1, 0, 0, 1, 0]
y_predict = [1, 0, 0, 0, 0, 0, 0, 0, 0, 0]

TPR = [0.16666667, 0.33333333, 0.33333333, 0.5, 0.66666667, 0.83333333, 0.83333333, 0.83333333, 1.0, 1.0]
FPR = [0.0, 0.0, 0.25, 0.25, 0.25, 0.25, 0.5, 0.75, 0.75, 1.0]

sklearn.metrics.roc_curve(y_true, y_score, pos_label=None, sample_weight=None, drop_intermediate=True)

def binary_clf_curve(y_true, y_score, pos_label=None):
    if pos_label is None:
        pos_label = 1.0

    # make y_true a boolean vector
    y_true = (y_true == pos_label)

    # sort scores
    desc_score_indices = np.argsort(y_score)[::-1]
    thresholds = y_score[desc_score_indices]

    fps = []
    tps = []
    for threshold in thresholds:
        # 大于等于阈值判定为 1 (正类)，否则为 0 (负类)
        y_predict = [1 if i >= threshold else 0 for i in y_score]
        # 预测值是否等于真实值
        result = [i == j for i, j in zip(y_true, y_predict)]
        # 预测值是否为 1 (正类)
        positive = [i == 1 for i in y_predict]

        # 预测为正类且预测错误
        fp = [(not i) and j for i, j in zip(result, positive)].count(True)
        # 预测为正类且预测正确
        tp = [i and j for i, j in zip(result, positive)].count(True)

        fps.append(fp)
        tps.append(tp)
    fps = np.array(fps)
    tps = np.array(tps)

    return fps, tps, thresholds


def roc_curve(y_true, y_score, pos_label=None):
    fps, tps, thresholds = binary_clf_curve(y_true, y_score, pos_label)

    fpr = fps / fps[-1]
    tpr = tps / tps[-1]

    return fpr, tpr, thresholds