1. 背景
在清洗事件的时候,需要调用接口来处理,我是直接一个ProcessFunction,来一条处理一条。问题就是出现请求接口的process反压的情况,如下图:
明明接口的qps能支持很高,时延也低,居然还反压。然后我就看看flink官网,可以使用异步的方式调用:
图中棕色的长格子就是等待时间,如果同步的方式调用接口,就会产生大量的等待时间,以至于我躲开并行度就不能解决反压问题。我的需求是清洗事件,下游用的也是事件事件,不需要顺序的请求,只要接口返回就算清洗完成,所以我可以使用flink异步io的方式处理。
2. 代码实现
函数调用
SingleOutputStreamOperator<UserDto> stream = AsyncDataStream.unorderedWait(
filterStream, //输入的数据流
new HttpAsync(), //异步查询的Function实例
6000, //超时时间
TimeUnit.MILLISECONDS, //时间单位
300 //最大异步并发请求数量(并发的线程队列数)
).disableChaining().setParallelism(6).name("interfaceProcess");
异步io函数
/**
* 异步接口调用
*/
@Slf4j
public class HttpAsyn extends RichAsyncFunction<UserDto, UserDto> {
CloseableHttpAsyncClient httpclient = null;
@Override
public void open(Configuration parameters) throws Exception {
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(3000) //设置HttpClient连接池超时时间
.build();
httpclient = HttpAsyncClients.custom()
.setMaxConnTotal(60) //并发连接最大数量
.setMaxConnPerRoute(60) //每个域名最大并发连接数量
.setDefaultRequestConfig(requestConfig)
.build();
httpclient.start();
}
@Override
public void asyncInvoke(UserDto userDto, ResultFuture<UserDto> resultFuture) throws Exception {
try {
//请求参数准备
HttpPost httpPost = new HttpPost();
httpPost.setHeader(BladeConstant.CONTENT_TYPE_NAME, BladeConstant.CONTENT_TYPE);
httpPost.setURI(new URI("接口地址"));
StringEntity stringEntity = new StringEntity(JSON.toJSONString(queryReq), "UTF-8");
httpPost.setEntity(stringEntity);
Future<HttpResponse> future = httpclient.execute(httpPost, null);
CompletableFuture<UserDto> completableFuture = CompletableFuture.supplyAsync(new Supplier<UserDto>() {
@Override
public UserDto get() {
try {
HttpResponse response = future.get();
if (null != response && response.getStatusLine().getStatusCode() == 200) {
HttpEntity entity = response.getEntity();
String rsp = EntityUtils.toString(entity);
String status = JSON.parseObject(rsp).getString("status");
if ("200".equals(status)){
userDto.setData(JSON.parseObject(rsp).getString("data"));
}
}
}catch (Exception e){
log.error("接口查询失败:{}", JSON.toJSONString(e));
}
}
});
completableFuture.thenAccept(new Consumer<UserDto>() {
@Override
public void accept(UserDto userDto) {
resultFuture.complete(Collections.singleton(userDto));
}
});
}catch (Exception e){
resultFuture.complete(Collections.singleton(null));
}
}
@Override
public void timeout(UserDto userDto, ResultFuture<UserDto> resultFuture) throws Exception {
log.error("超时记录:{}" + userDto);
}
}