读取二进制文件（后缀.bin)，解析二进制文件，并入库

本文链接：https://blog.csdn.net/javahelpyou/article/details/107985521

本文介绍了一种使用Flink处理大型二进制文件的方法，该文件包含振动数据，通过自定义源函数读取数据并进行flatMap转换，将解析后的数据以特定格式输出，并最终存储到Redis中。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

1.场景

提供了一个接近1G的二进制文件，内容为振动数据（包含头数据、有效数据、尾数据），要求解析入库

2.思路

运行在flink上，从文件中增量抽数据，flatmap处理，最后sink到redis中

public class SourceFromFile extends RichSourceFunction<byte[]> {
    private volatile Boolean isRunning = true;

    @Override
    public void run(SourceContext ctx) throws Exception {

        File file = new File("/root/Ca.bin");
        FileInputStream fi = new FileInputStream(file);
        byte[] buffer = new byte[413264];
        fi.read(new byte[10]);
        int count=0;
        while (isRunning) {
            int read = fi.read(buffer, 0, 413264);

            if(read==413264){
                ctx.collect(buffer);

            }else {
                isRunning=false;
            }
        }
        fi.close();
    }


    @Override
    public void cancel() {
        isRunning = false;
    }
}

public class ReadF {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        DataStreamSource<byte[]> dataStream = env.addSource(new SourceFromFile());
        SingleOutputStreamOperator<Tuple3<String, String, Object>> flatMap = dataStream.flatMap(new FlatMapFunction<byte[], Tuple3<String, String, Object>>() {
            @Override
            public void flatMap(byte[] buf, Collector<Tuple3<String, String, Object>> out) throws Exception {
                StringBuffer sb = new StringBuffer();
                for (int i = 0; i < 3; i++) {
                    String hex = Integer.toHexString(buf[i] & 0xFF);
                    if (hex.length() == 1) {
                        hex = '0' + hex;
                    }
                    sb.append(hex.toUpperCase());
                }
                String s = UUID.randomUUID().toString();
                Tuple3<String, String, Object> modify = Tuple3.of(s, "modify", sb.toString());
                out.collect(modify);
                Tuple3<String, String, Object> num = Tuple3.of(s, "num", (buf[3]&0xFF) + "");
                out.collect(num);
                Tuple3<String, String, Object> car = Tuple3.of(s, "car", (buf[5] & 0xFF)+ "");
                out.collect(car);
                Tuple3<String, String, Object> sign = Tuple3.of(s, "sign", (buf[6]& 0xFF) + "");
                out.collect(sign);
                Tuple3<String, String, Object> time = Tuple3.of(s, "time", (2000 + (buf[7]& 0xFF)) + "年" +  (buf[8]& 0xFF) + "月" + (buf[9]& 0xFF) + "日" + (buf[10]& 0xFF) + "时" + (buf[11]& 0xFF) + "分" + (buf[12]& 0xFF) + "秒");
                out.collect(time);
                ArrayList<Integer> list = new ArrayList<>();
                for (int i = 1; i < 409933 / 16; i++) {
                    int sum=0;
                   for(int j=0;j<16;j++){
                       sum+=(buf[16*i+j]& 0xFF);
                   }
                   list.add(sum);
                }
                Tuple3<String, String, Object> of = Tuple3.of(s, "振动" , list);
                out.collect(of);
            }
        }).setParallelism(2);
        flatMap.addSink(new MyRedisSink());
        env.execute();

    }
}

class MyRedisSink extends RichSinkFunction<Tuple3<String, String, Object>> {
    private transient Jedis jedis;

    @Override
    public void open(Configuration config) {
        jedis = new Jedis("192.168.1.127", 16379, 20000);
        jedis.select(2);
    }

    @Override
    public void invoke(Tuple3<String, String, Object> value, Context context) {
        if (!jedis.isConnected()) {
            jedis.connect();
        }
        //保存
        jedis.hset(value.f0, value.f1, String.valueOf(value.f2));
    }

    @Override
    public void close() {
        jedis.close();
    }

    public static void main(String[] args) {
        long l = System.currentTimeMillis();
        Jedis jedis = new Jedis("192.168.1.127", 16379, 20000);
        jedis.select(2);
        String hget = jedis.hget("test", "振动");
        ArrayList list=new ArrayList(Arrays.asList(hget.split(",")));
        String s = (String) list.get(2);
        System.out.println(s);
        long l1 = System.currentTimeMillis();
        System.out.println(l1-l);
    }

}