【Python脚本】AI系统开发常用的Python脚本合集

json读取文件代码

import json


def load_json_items(json_path):  # 多个dict
    with open(json_path, "r") as f:
        json_items = [json.loads(s) for s in f.readlines()]
    return json_items


def save_json_items(json_path, json_items):
    with open(json_path, "w") as f:
        f.write("\n".join([json.dumps(x) for x in json_items]))


def load_json_data(json_path):   # 单个dict
    with open(json_path, "r") as f:
        json_data = json.load(f)
    return json_data


def save_json_data(json_path, json_data):
    with open(json_path, "w") as f:
        json.dump(json_data, f)

检查并创建文件夹

def check_and_make_dir(dir_path):
    if os.path.exists(dir_path):
        print("{} already exists!!!".format(dir_path))
        #exit()
    else:
        os.makedirs(dir_path)
        assert os.path.exists(dir_path), dir_path

check_and_make_dir(save_dir)  # 建立文件夹

多进程代码模板

from tqdm import tqdm
from multiprocessing import Queue, Process


def split_task_for_workers(records, num_worker):
    """
    给每个worker分配要处理的数据
    """
    splits = {}
    for i_worker in range(num_worker):
        splits[i_worker] = []
    for i_data in range(len(records)):
        splits[i_data % num_worker].append(records[i_data])
    return splits


def compute_for_list(worker_id, records, *, paral_queue=None):
    for record in records:
        y = record ** 2
        paral_queue.put(y)


def handle(xs, num_worker):
    nr_records = len(xs)
    pbar = tqdm(total=nr_records)   # 进度条
    paral_queue = Queue()
    procs = []
    
    worker_records = split_task_for_workers(xs, num_worker)

    for i_worker in range(num_worker):
        records = worker_records[i_worker]
        proc = Process(target=compute_for_list,
                       args=(i_worker, records),
                       kwargs=dict(
                           paral_queue=paral_queue,
                       ))
        proc.start()
        procs.append(proc)

    ys = []
    for i in range(nr_records):
        y = paral_queue.get()
        ys.append(y)
        pbar.update(1)

    for p in procs:
        p.join()

    return ys


def main():
    xs = list(range(0, 101))  # 输入参数
    num_worker = 16           # 进程数量

    ys = handle(xs, num_worker)

    ys.sort()

    print("xs:")
    print(xs)
    print("ys:")
    print(ys)


if __name__ == "__main__":
    main()

多个json文件合并为一个

    root_dir = './'
    out_sds = './data_all.sds'
    paths = os.listdir(root_dir)

    count = 0 #记录数据总条数

    with open(out_sds,"w+") as f:
        for path in paths:
            file_path  = os.path.join(root_dir, path) 
            print(file_path)
            sds_name, suffix = os.path.splitext(path)
            if (suffix != '.sds') | (sds_name == 'data_all'):
                continue
            with open(file_path, 'r', encoding='utf8')as fp:
                for line in fp:
                    data = json.loads(line)  # 字典类型
                    count = count + 1
                    #print(data)
                    #print('这是读取到文件数据的数据类型：', type(data))
                    print(count)
                    json.dump(data, f, ensure_ascii=False)  # 写入文件，ensure_ascii=False避免将中文转化为编码
                    f.write('\r\n')
                    #print(file_path + "加载入文件完成...")
    

项目文件目录下的加载路径

• 先上示例：

import os
import sys
__dir__ = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, __dir__)

• 解释一下：
  1.os.path.abspath()：

os.path.abspath(__file__) 功能： 获取当前脚本的完整路径

注意：os.path.abspath(file)返回的是脚本的绝对路径，即setting.py文件的绝对路径。

2.os.path.dirname()：

os.path.dirname(path)  功能：去掉文件名，返回目录

注意：os.path.dirname(path) 返回path的父路径；可嵌套使用，os.path.dirname(os.path.dirname(path) ) 返回父路径的父路径。

3.os.path.basename(file)：

os.path.basename(__file__)  功能：返回脚本的文件名称

4.sys.path.insert(0, “/path”)

sys.path.insert(0, __dir__)

可以选择用sys.path.insert(0,‘/path’)，这样新添加的目录会优先于其他目录被import检查

手动过滤图片数据样本的自动化操作

可以通过键盘快速操作

import cv2
import os
import argparse
import numpy as np 
from tqdm import tqdm
import shutil
import random
import glob
import json
import sys
import tty
import termios

def parse_arg():
    """
    Parse command line input parameters
    """
    parser = argparse.ArgumentParser(description="Calculate the accuracy of license plate recognition")
    parser.add_argument("--input_image_paths", type=str, default="/home/yangbinchao/桌面/雄研/draw_test_data_image/draw",help=" File path to be processed ")
    args = parser.parse_args()
    print()
    print(f"args = {args}")
    print()
    return args

def readchar():
    fd = sys.stdin.fileno()
    old_settings = termios.tcgetattr(fd)
    try:
        tty.setraw(sys.stdin.fileno())
        ch = sys.stdin.read(1)
    finally:
        termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
    return ch

def readkey(getchar_fn=None):
    getchar = getchar_fn or readchar
    c1 = getchar()
    if ord(c1) != 0x1b:
        return c1
    c2 = getchar()
    if ord(c2) != 0x5b:
        return c1
    c3 = getchar()
    return chr(0x10 + ord(c3) - 65)

def wait_key(img_path):  # wait the key is put down
    while True:
        key=readkey()
        if key=='z':  # save the path 
            print('save the image path list',img_path)
            #miss_label_list.append(img_path)
            with open( './miss_label_list.json',  'a+',  encoding='utf-8') as f:  # 车牌对应的颜色标签
                f.write(json.dumps(img_path,ensure_ascii=False,indent=1)+',')
            break
        if key=='x':  
            print('save the image path list',img_path)
            #lack_vehicle_list.append(img_path)
            with open( './lack_vehicle_list.json',  'a+',  encoding='utf-8') as f:  # 车牌对应的颜色标签
                f.write(json.dumps(img_path,ensure_ascii=False,indent=1)+',')
            break
        if key=='a':
            print('ignore the image path list',img_path)
            break
        if key=='q':
            return key
            break

def read_img(img_path,image_dir):
    image_path = os.path.join( image_dir, img_path)
    img = cv2.imread(image_path) 
    img = cv2.resize(img,(1000,500))
    cv2.imshow(img_path, img) 
    while cv2.waitKey(2500) != 27:# loop time is 3000 if not get ESC
        if cv2.getWindowProperty('img_path' ,cv2.WND_PROP_VISIBLE) <= 0:
            break
    cv2.destroyAllWindows() 

def main(args):

    for img_path in tqdm(os.listdir(args.input_image_paths)):
        read_img(img_path, image_dir=args.input_image_paths)
        key_value = wait_key(img_path)
        if(key_value == 'q'):
            print('\nthe save image path list is {}'.format(miss_label_list))
            print('\nthe save image path list is {}'.format(lack_vehicle_list))
            exit()
    print('\nthe save image path list is {}'.format(miss_label_list))
    print('\nthe save image path list is {}'.format(lack_vehicle_list))

if __name__ == "__main__":
    args = parse_arg()
    miss_label_list = []
    lack_vehicle_list = []
    main(args)