Flink学习之 - map与flatMap

最新推荐文章于 2025-06-21 10:25:26 发布

.简.单.

最新推荐文章于 2025-06-21 10:25:26 发布

阅读量990

点赞数

CC 4.0 BY-SA版权

分类专栏： flink 文章标签： flink kafka 大数据

本文链接：https://blog.csdn.net/sasa527/article/details/121001184

flink 专栏收录该内容

6 篇文章

订阅专栏

本文介绍了如何使用Apache Flink进行实时数据流处理，从Kafka消费数据，筛选billCode以'test'开头的数据，然后加入当前时间戳，并处理成JSONObject。最后演示了如何将数据写入HBase并展示数据转换过程。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >


import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import org.apache.flink.api.common.functions.FlatMapFunction;
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.functions.RichMapFunction;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.util.Collector;
import org.apache.hadoop.hbase.client.Connection;
import java.util.Map;

public class MyFlinkTest {
    public static void main(String[] args) throws Exception {
      
        StreamExecutionEnvironment env = FlinkUtils.getStreamExecutionEnvironment();
        env.setParallelism(10);

        DataStreamSource<String> kafkaStream = env
                .addSource(KafkaUtil.getFlinkKafkaConsumer("dwd_table", "dwdTableTest"));

        kafkaStream.rebalance().map(new MapFunction<String, Object>() {
            private static final long serialVersionUID = 1L;
            public String map(String value){
                System.out.println("src value:"+value);
                return null;
            }
        }).uid("src.print").name("src.print.name").setParallelism(1);

        //只取billCode为test开头的数据
        SingleOutputStreamOperator<JSONObject> processStream = dataProcess(kafkaStream);

        //增加当前时间字段
        SingleOutputStreamOperator<JSONObject>  curTimeStream = curTimeDataProcess(kafkaStream);

        curTimeStream.map(new MapFunction<JSONObject,Object>() {
            @Override
            public Object map(JSONObject jsonObject) throws Exception {
                System.out.println("json value:"+jsonObject);
                return null;
            }
        }).uid("json.print").name("json.print.name").setParallelism(2);

        env.execute("dwdTableTest");

    }

    private static SingleOutputStreamOperator<JSONObject> dataProcess(DataStreamSource<String> kafkaStream) {
        return kafkaStream.flatMap(new FlatMapFunction<String, JSONObject>() {
            @Override
            public void flatMap(String value, Collector<JSONObject> collector) throws Exception {
                JSONObject source = JSON.parseObject(value);
                if(source.getString("billCode").startsWith("test")){
                    collector.collect(source);
                }
            }
        }).uid("billCode.test").name("billCode.test.name").setParallelism(4);
    }


    private static SingleOutputStreamOperator<JSONObject> curTimeDataProcess(DataStreamSource<String> kafkaStream) {
        return kafkaStream.map(new MapFunction<String, JSONObject>() {
            @Override
            public JSONObject map(String s) throws Exception {
                JSONObject source = JSON.parseObject(s);
                source.put("curTime", DateUtils.getLocalDateTimeStr());
                return source;
            }
        }).uid("curTime.test").name("curTime.test.name").setParallelism(4);
    }


    private static SingleOutputStreamOperator<MyTestDTO> richMapProcess1(DataStreamSource<String> kafkaStream) {
        return kafkaStream.map(new RichMapFunction<String, MyTestDTO>() {

            private Connection conn;

            @Override
            public void open(Configuration parameters) throws Exception {
                conn = HbaseUtils.getConn();
            }

            @Override
            public void close() throws Exception {
                HbaseUtils.close(conn);
            }

            @Override
            public MyTestDTO map(String s) throws Exception {
                MyTestDTO myTestDTO = new MyTestDTO();
                myTestDTO.setDate("2021-10-27");
                myTestDTO.setWeight(1);
                return myTestDTO;
            }

        }).uid("richMapProcess1").setParallelism(4);
    }

    private static SingleOutputStreamOperator<String> richMapProcess2(DataStreamSource<JSONObject> kafkaStream) {
        return kafkaStream.map(new RichMapFunction<JSONObject, String>() {

            @Override
            public String map(JSONObject jSONObject) throws Exception {
                return jSONObject.toJSONString();
            }

        }).uid("richMapProcess2").setParallelism(4);
    }

    private static SingleOutputStreamOperator<MyTestDTO> richMapProcess3(DataStreamSource<Map<String,String>> kafkaStream) {
        return kafkaStream.map(new RichMapFunction<Map<String, String>, MyTestDTO>() {

            @Override
            public MyTestDTO map(Map<String, String> stringStringMap) throws Exception {
                return null;
            }

        }).uid("richMapProcess3").setParallelism(4);
    }

}

import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;

@Data
@NoArgsConstructor
@AllArgsConstructor
public class MyTestDTO {
    private String date;
    private double weight;
}