一次消息消费服务的内存泄漏排查小记

本文链接：https://blog.csdn.net/hiwatershed/article/details/113855758

某在线消息消费服务使用spring-kafka遇到内存泄漏问题，表现为老年代内存无法释放，通过堆栈分析发现大量ThreadLocalScope对象及分布式追踪库dd-trace-java中的FakeSpan类存在内存泄漏。分析代码后，发现在处理Kafka消息时，子线程创建的ThreadLocalScope因循环引用无法被垃圾回收，导致内存积压。主线程创建的ThreadLocalScope则能正常回收，问题出在RecoredValueAdvice和BatchMessageListenerAdvice之间的协调不一致。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

线上有一个消息消费服务xxx-consumer，使用spring-kafka框架，主线程批量从消费队列（kafka）拉取交易系统生产的消息，然后提交到子线程池中挨个处理消费。

public abstract class AbstractMessageDispatchListener implements
        BatchAcknowledgingMessageListener<String, Msg>, ApplicationListener<ApplicationReadyEvent> 
{

    private ThreadPoolExecutor executor;

    public abstract MessageWorker chooseWorker(ConsumerRecord<String, Msg> data);

    @Override
    public void onMessage(List<ConsumerRecord<String, Msg>> datas, Acknowledgment acknowledgment) {
        List<Future<?>> futureList = new ArrayList<>(datas.size());
        try {
            CountDownLatch countDownLatch = new CountDownLatch(datas.size());
            for (ConsumerRecord<String, Msg> data : datas) {
                Future<?> future = executor.submit(new Worker(data, countDownLatch));
                futureList.add(future);
            }

            countDownLatch.await(20000L - 2000, TimeUnit.MILLISECONDS);
            long countDownLatchCount = countDownLatch.getCount();
            if (countDownLatchCount > 0) {
                return;
            }
            acknowledgment.acknowledge();
        } catch (Exception e) {
            logger.error("onMessage error ", e);
        } finally {
            for (Future<?> future : futureList) {
                if (future.isDone() || future.isCancelled()) {
                    continue;
                }
                future.cancel(true);
            }
        }
    }

    @Override
    public void onApplicationEvent(ApplicationReadyEvent event) {
        ThreadFactoryBuilder builder = new ThreadFactoryBuilder();
        builder.setNameFormat(this.getClass().getSimpleName() + "-pool-%d");
        builder.setDaemon(false);
        executor = new ThreadPoolExecutor(12,
                12 * 2,
                60L,
                TimeUnit.SECONDS,
                new ArrayBlockingQueue<>(100),
                builder.build());
    }

    private class Worker implements Runnable {
        private ConsumerRecord<String, Msg> data;
        private CountDownLatch countDownLatch;

        Worker(ConsumerRecord<String, Msg> data, CountDownLatch countDownLatch) {
            this.data = data;
            this.countDownLatch = countDownLatch;
        }

        @Override
        public void run() {
            try {
                MessageWorker worker = chooseWorker(data);
                worker.work(data.value());
            } finally {
                countDownLatch.countDown();
            }
        }
    }