最近在在自己的数据集上跑一些目标检测的网络，所以需要把数据集处理成符合网络输入的格式。其中一个网络是要将图片的真值标签按照图片的顺序写入json文件，我的数据集图片名称为0.jpg、1.jpg、2.jpg……，标签文件为0.txt、1.txt、2.txt……，然后用os.listdir读取标签文件夹下的所有文件，对应的图片序号我当时直接for循环里每次加1得到（这就导致标签和图片不对应了）。最终生成的json文件中标签和图片的对应变成了：
| 标签 | 图片 |
|---|---|
| 0.txt | 0.txt |
| 1.txt | 1.txt |
| 10.txt | 2.txt |
| 100.txt | 3.txt |
| 1000.txt | 4.txt |
当我用这样的标签训练完网络后在测试集上测试时，检测出的目标非常混乱，完全没有目标的地方也判断为了目标，然后倒回来查看数据集的时候才发现了问题所在。在原本os.listdir后增加了排序的一行代码，这样标签和图片就可以对应上了。
ann_list = os.listdir(ann_root)
ann_list.sort(key=lambda x: int(x[:-4])) ### 新加的，否则写入json的文件顺序为0、1、10、100、1000.。。
血的教训啊，以后用os.listdir的时候一定要小心！！！
最后附上当时生成json文件的代码，修改的地方在read_all_labels函数中：
import os
import sys
from pathlib import Path
import numpy as np
sys.path.append(os.path.abspath(Path(__file__).parent.parent))
import shutil
import cv2
import json
from tqdm import tqdm
## 读取图像，解决imread不能读取中文路径的问题
def cv_imread(filePath):
cv_img=cv2.imdecode(np.fromfile(filePath,dtype=np.uint8),-1)
return cv_img
def read_label_txt(txt_file):
f = open(txt_file, 'r')
lines = f.readlines()
labels = []
for line in lines:
line = line.strip().split(',')
x, y, w, h= line[:4]
labels.append(
{'bbox': (int(float(x)),int(float(y)),int(w),int(h)),
'ignore': 0,
'class': 1,
'truncate': 0, ### 没用到
'occlusion': 0} ### 没用到
)
return labels
def read_all_labels(ann_root):
ann_list = os.listdir(ann_root)
ann_list.sort(key=lambda x: int(x[:-4])) ### 新加的，否则写入json的文件顺序为0、1、10、100、1000.。。
all_labels = {}
for ann_file in ann_list:
if not ann_file.endswith('txt'):
continue
ann_labels = read_label_txt(os.path.join(ann_root, ann_file))
all_labels[ann_file] = ann_labels
return all_labels
def get_save_path(img_path, index):
name = img_path.split('.')[0]
return name + '.jpg'
def crop_and_save_image(img_root, img_path, new_img_root):
img = cv_imread(os.path.join(img_root, img_path))
h, w = img.shape
_y = h // 2
_x = w // 2
return h, w, _y, _x
def copy_image(img_root, img_path, new_img_root):
img = cv_imread(os.path.join(img_root, img_path))
h, w = img.shape
cv2.imwrite(os.path.join(new_img_root, img_path), img)
return h, w
def get_new_label(label, img_path, cy, cx, id, img_id_base):
if label['class'] == 0 or label['ignore']:
return None
x, y, w, h = label['bbox']
nx, ny, nw, nh = x, y, w, h
img_id = img_id_base
new_label = {'category_id': label['class'], 'id': id, 'iscrowd': 0, 'image_id': img_id, 'area': nw * nh,
'segmentation': [], 'bbox': [nx, ny, nw, nh]}
return new_label
def label_to_coco(label, id, img_id):
x, y, w, h = label['bbox']
new_label = {'category_id': label['class'], 'id': id, 'iscrowd': 0, 'image_id': img_id, 'area': w * h,
'segmentation': [], 'bbox': [x, y, w, h]}
return new_label
def make_json(images, annotations, new_label_json):
ann_dict = {}
ann_dict['categories'] = [
{'supercategory': 'things', 'id': 1, 'name': 'target'},
]
ann_dict['images'] = images
ann_dict['annotations'] = annotations
with open(new_label_json, 'w') as outfile:
json.dump(ann_dict, outfile)
def make_new_train_set(img_root, label_root, new_img_root, new_label_json):
all_labels = read_all_labels(label_root)
annotations = []
images = []
ann_id = 0
img_id = 0
for filename, labels in tqdm(all_labels.items()):
img_path = filename.replace('txt', 'jpg')
h, w, cy, cx = crop_and_save_image(img_root, img_path, new_img_root)
images.append({'file_name': get_save_path(img_path, 0), 'height': h, 'width': w, 'id': img_id})
for label in labels:
new_label = get_new_label(label, img_path, cy, cx, ann_id, img_id)
if new_label != None:
ann_id += 1
annotations.append(new_label)
# img_id += 4
img_id += 1
make_json(images, annotations, new_label_json)
def make_new_train_set_trainval2017(img_root, label_root, new_img_root, new_label_json):
all_labels = read_all_labels(label_root)
annotations_train = []
images_train = []
annotations_val = []
images_val = []
ann_id = 0
img_id = 0
for filename, labels in tqdm(all_labels.items()):
img_path = filename.replace('txt', 'jpg')
h, w, cy, cx = crop_and_save_image(img_root, img_path, new_img_root)
val_index = int(img_path.split('.')[0]) ### 原本下面一行的判断用的是img_id，导致验证集并非我想要的那些
if (val_index >= 4015 and val_index <= 5014) or (val_index >= 13015 and val_index <= 14014) or (val_index >= 24832 and val_index <= 25831) or (val_index >= 27832 and val_index <= 28831):
images_val.append({'file_name': get_save_path(img_path, 0), 'height': h, 'width': w, 'id': img_id})
for label in labels:
new_label = get_new_label(label, img_path, cy, cx, ann_id, img_id)
if new_label != None:
ann_id += 1
annotations_val.append(new_label)
else:
images_train.append({'file_name': get_save_path(img_path, 0), 'height': h, 'width': w, 'id': img_id})
for label in labels:
new_label = get_new_label(label, img_path, cy, cx, ann_id, img_id)
if new_label != None:
ann_id += 1
annotations_train.append(new_label)
img_id += 1
make_json(images_val, annotations_val, new_label_json.replace('train2017','val2017'))
make_json(images_train, annotations_train, new_label_json)
def make_new_test_set(img_root, label_root, new_img_root, new_label_json):
all_labels = read_all_labels(label_root)
annotations = []
images = []
ann_id = 0
img_id = 0
for filename, labels in tqdm(all_labels.items()):
img_path = filename.replace('txt', 'jpg')
h, w = copy_image(img_root, img_path, new_img_root)
images.append({'file_name': img_path, 'height': h, 'width': w, 'id': img_id})
for label in labels:
coco_label = label_to_coco(label, ann_id, img_id)
if coco_label != None:
ann_id += 1
annotations.append(coco_label)
img_id += 1
make_json(images, annotations, new_label_json)
if __name__ == '__main__':
'''
Training
'''
img_root = r'D:MINEprojectIRSTdatasetdataset(combine)trainimages'
label_root = r'D:MINEprojectIRSTdatasetdataset(combine)trainlabels(QueryDet)'
new_img_root = r'D:MINEprojectIRSTdatasetdataset(combine)trainimages'
new_label_json = r'D:MINEprojectIRSTdatasetdataset(combine)annotationsinstances_train2017.json'
make_new_train_set_trainval2017(img_root, label_root, new_img_root, new_label_json) ### train和val一起获得了
