多线程事务完成百万级数据插入

直接上工具类：

@Slf4j
@Component
public class Utils{

    //spring的事务管理器，用于控制事务的提交和回滚
    private  static PlatformTransactionManager platformTransactionManager;
   


    @Autowired
    public  Utils( PlatformTransactionManager platformTransactionManager) {
        MoreThreadTransactionUtils.platformTransactionManager = platformTransactionManager;
    }


    public  static <T> boolean execute(List<T> data, Consumer<List<T>> consumer){
        List<Runnable> tasks = new ArrayList<>();
        List<List<T>> spilt = spilt(data, 50000);
        for (List<T> ts : spilt) {
            tasks.add(() -> consumer.accept(ts));
        }
        return executeTransaction(tasks);
    }


    /**
     * 执行
     */
    public static boolean executeTransaction( List<Runnable> taskList) {
        if (taskList == null || taskList.size()== 0) {
            throw new IllegalArgumentException("taskList is empty");
        }
      
        int taskSize = taskList.size();
      
        AtomicInteger taskSuccessAccount = new AtomicInteger(0);

        List<Future<?>> taskFutureList = new ArrayList<>(taskSize);
   
        CyclicBarrier cyclicBarrier = new CyclicBarrier(taskSize);
        int i = 1;
        ExecutorService executorService = Executors.newFixedThreadPool(taskSize);
        try {
          
            for (Runnable task : taskList) {
                final int taskIndex = i;
                Future<?> future = executorService.submit(() -> {
                    TransactionStatus transactionStatus = null;
                    try {
                       
                        transactionStatus = platformTransactionManager.getTransaction(new DefaultTransactionAttribute());
                 
                        task.run();
                        // 成功数量+1
                        taskSuccessAccount.incrementAndGet();
                        log.debug("task：{} 等待提交事务", taskIndex);
                    } catch (Throwable e) {
                        log.error("task：{}，执行异常，异常原因：{}", taskIndex, e.getMessage());
                    } finally {
                        try {
                            cyclicBarrier.await();
                        } catch (Exception e) {
                            log.error("cyclicBarrier.await error:{}", e.getMessage(), e);
                        }
                    }
                    if (transactionStatus != null) {
                       
                        if (taskSuccessAccount.get() == taskSize) {
                            // 成功，提交事务
                            log.debug("task：{} 提交事务", taskIndex);
                            platformTransactionManager.commit(transactionStatus);
                        } else {
                            //失败，回滚事务
                            log.debug("task：{} 回滚事务", taskIndex);
                            platformTransactionManager.rollback(transactionStatus);
                        }
                    }
                });
                taskFutureList.add(future);
                i++;
            }
            for (Future<?> future : taskFutureList) {
                try {
                    future.get();
                } catch (Exception e) {
                    log.error("future.get error:{}", e.getMessage(), e);
                }
            }
        } finally {
           
            executorService.shutdown();
        }
        return taskSuccessAccount.get() == taskSize;
    }
    /**
     * @param needSpiltList 需要拆分的集合
     * @param spiltSize 每个集合大小
     * @return 拆分后都集合
     */
    public static <T> List<List<T>> spilt(List<T> needSpiltList, int spiltSize){
        //防御性编程
        if (CollectionUtils.isEmpty(needSpiltList) || spiltSize <=0){
            return new ArrayList<>();
        }
        //结果
        List<List<T>> result = new ArrayList<>();
        //集合大小
        int size = needSpiltList.size();
        //最多分一批次
        if (size<=spiltSize){
            result.add(needSpiltList);
            return result;
        }
        //出最后一个，每批次大小
        int preSize = size / spiltSize;
        //最后一个批次大小
        int lastSize = size % spiltSize;
        //开始标记
        int beginIndex = 0;
        //结束标记
        int endIndex = 0;
        for (int i = 0; i < preSize; i++) {
            endIndex = beginIndex + spiltSize;
            //最后一批，用最后一批的
            if (i == preSize-1 && lastSize != 0){
                endIndex = beginIndex + lastSize;
            }
            result.add(needSpiltList.subList(beginIndex,endIndex));
            //移动游标
            beginIndex = beginIndex + spiltSize;
        }
        return result;
    }
}

具体使用：

boolean success =  Utils.execute(data,
                (data)-> this.testMapper.batchInsert("test", data));

要注意插入方法，如果每个线程事务执行的数据过大，就需要插入时的分批次

       List<List<TestPO>> spilt = Utils.spilt(list, 3000);
        for (List<TestPO> testPOS : spilt) {
            this.batchInsert(tableName, testPOS);
        }