Netty对HTTP2 Frame编解码

前言

在HTTP/1.x时代，HTTP是一个文本协议，简单直接，人类友好的可读性，头部和主体通过换行符来区分，计算机处理起来低效且容易出错。
HTTP2打破了这个传统，它不改变HTTP协议的语义，请求响应仍然有头部和主体，只是在数据传输上做了改变，由之前的文本改为二进制分帧传输。

Frame

通过换行符来区分头部和主体，低效且容易出错。为此HTTP2设计了多种不同类型的Frame，用来传输不同类型的数据，Frame被设计的职责清晰，边界明确。例如：传输头部的HEADERSFrame、传输主体数据的DATAFrame。
HTTP2 Frame由9字节定长的Header和变长的Payload组成，格式如下：

不同的Frame装载的Payload格式也不尽相同，以HEADERS和DATAFrame为例：

DATA Frame {
  Length (24),
  Type (8) = 0x00,

  Unused Flags (4),
  PADDED Flag (1),
  Unused Flags (2),
  END_STREAM Flag (1),

  Reserved (1),
  Stream Identifier (31),

  [Pad Length (8)],
  Data (..),
  Padding (..2040),
}

DATAPayload很简单，只有数据本身和填充字节。

HEADERS Frame {
  Length (24),
  Type (8) = 0x01,

  Unused Flags (2),
  PRIORITY Flag (1),
  Unused Flag (1),
  PADDED Flag (1),
  END_HEADERS Flag (1),
  Unused Flag (1),
  END_STREAM Flag (1),

  Reserved (1),
  Stream Identifier (31),

  [Pad Length (8)],
  [Exclusive (1)],
  [Stream Dependency (31)],
  [Weight (8)],
  Field Block Fragment (..),
  Padding (..2040),
}

HEADERSPayload就复杂一些，除了Field Block Fragment存储Headers之外，还有用来设置流依赖关系、权重等属性。

Http2FrameReader

Netty提供了接口io.netty.handler.codec.http2.Http2FrameReader来从输入缓冲区读取HTTP2 Frame，让开发者不用关心Frame的底层细节，专注于业务。
Http2FrameReader核心方法是readFrame()，它会从输入缓冲区input读取字节序列并转换为Frame，然后触发监听器。

public interface Http2FrameReader extends Closeable {

    void readFrame(ChannelHandlerContext ctx, ByteBuf input, 
                   Http2FrameListener listener);
}

针对不同类型的Frame，Http2FrameListener提供了对应的监听方法。

public interface Http2FrameListener {

    int onDataRead(ChannelHandlerContext ctx, int streamId, ByteBuf data, int padding,
                   boolean endOfStream) throws Http2Exception;

    void onHeadersRead(ChannelHandlerContext ctx, int streamId, Http2Headers headers, int padding,
            boolean endOfStream) throws Http2Exception;

    void onHeadersRead(ChannelHandlerContext ctx, int streamId, Http2Headers headers,
            int streamDependency, short weight, boolean exclusive, int padding, boolean endOfStream)
            throws Http2Exception;

    void onPriorityRead(ChannelHandlerContext ctx, int streamId, int streamDependency,
            short weight, boolean exclusive) throws Http2Exception;

    void onRstStreamRead(ChannelHandlerContext ctx, int streamId, long errorCode) throws Http2Exception;

    void onSettingsAckRead(ChannelHandlerContext ctx) throws Http2Exception;

    void onSettingsRead(ChannelHandlerContext ctx, Http2Settings settings) throws Http2Exception;

    void onPingRead(ChannelHandlerContext ctx, long data) throws Http2Exception;

    void onPingAckRead(ChannelHandlerContext ctx, long data) throws Http2Exception;

    void onPushPromiseRead(ChannelHandlerContext ctx, int streamId, int promisedStreamId,
            Http2Headers headers, int padding) throws Http2Exception;

    void onGoAwayRead(ChannelHandlerContext ctx, int lastStreamId, long errorCode, ByteBuf debugData)
            throws Http2Exception;

    void onWindowUpdateRead(ChannelHandlerContext ctx, int streamId, int windowSizeIncrement)
            throws Http2Exception;

    void onUnknownFrame(ChannelHandlerContext ctx, byte frameType, int streamId, Http2Flags flags, ByteBuf payload)
            throws Http2Exception;
}

Http2FrameReader有一个默认实现DefaultHttp2FrameReader，因为TCP粘包拆包等原因，Netty采用循环读的方式，只要有数据可读就会一直读，直到读取不到完整的Frame才会等待对方继续发送数据包。
Frame由9字节header和变长的Payload组成，Netty先读取并校验header，再读取Payload，再触发监听器。

@Override
public void readFrame(ChannelHandlerContext ctx, ByteBuf input, Http2FrameListener listener)
        throws Http2Exception {
    if (readError) {
        input.skipBytes(input.readableBytes());
        return;
    }
    try {
        // 循环读 因为粘包拆包，可能有多个Frame
        do {
            // 先读header，读取到完整header再尝试读payload
            if (readingHeaders) {
                processHeaderState(input);// 读取header并校验
                if (readingHeaders) {
                    // header没读完，等待对方继续发送数据包
                    return;
                }
            }
            // 读取payload
            processPayloadState(ctx, input, listener);
            if (!readingHeaders) {
                return;
            }
        } while (input.isReadable());
    } catch (Http2Exception e) {
        readError = !Http2Exception.isStreamError(e);
        throw e;
    } catch (RuntimeException e) {
        readError = true;
        throw e;
    } catch (Throwable cause) {
        readError = true;
        PlatformDependent.throwException(cause);
    }
}

DefaultHttp2FrameReader#processHeaderState()：按照Frame规定的格式读取header，并校验。

private void processHeaderState(ByteBuf in) throws Http2Exception {
    if (in.readableBytes() < FRAME_HEADER_LENGTH) {
        // header不完整，等待继续传输数据包
        return;
    }
    // 前24Bit是Payload长度
    payloadLength = in.readUnsignedMedium();
    if (payloadLength > maxFrameSize) {
        throw connectionError(FRAME_SIZE_ERROR, "Frame length: %d exceeds maximum: %d", payloadLength,
                              maxFrameSize);
    }
    // 读取Frame Type
    frameType = in.readByte();
    // 读取Flags
    flags = new Http2Flags(in.readUnsignedByte());
    // 读取 StreamID
    streamId = readUnsignedInt(in);
    // header读取完毕，设为false，后续读取Payload
    readingHeaders = false;
    // 校验header
    switch (frameType) {
        case DATA:
            verifyDataFrame();
            break;
        case HEADERS:
            verifyHeadersFrame();
            break;
        case PRIORITY:
            verifyPriorityFrame();
            break;
        case RST_STREAM:
            verifyRstStreamFrame();
            break;
        case SETTINGS:
            verifySettingsFrame();
            break;
        case PUSH_PROMISE:
            verifyPushPromiseFrame();
            break;
        case PING:
            verifyPingFrame();
            break;
        case GO_AWAY:
            verifyGoAwayFrame();
            break;
        case WINDOW_UPDATE:
            verifyWindowUpdateFrame();
            break;
        case CONTINUATION:
            verifyContinuationFrame();
            break;
        default:
            // 未知的Frame类型
            verifyUnknownFrame();
            break;
    }
}

DefaultHttp2FrameReader#processPayloadState()：读取Payload，并触发监听器。因为不同类型的Frame装载的Payload格式是不一样的，因此Netty提供了很多方法来读取这些Payload。

private void processPayloadState(ChannelHandlerContext ctx, ByteBuf in, Http2FrameListener listener)
                throws Http2Exception {
    if (in.readableBytes() < payloadLength) {
        // Payload不完整，等待对方发送数据包
        return;
    }

    // 记录Payload结束索引
    int payloadEndIndex = in.readerIndex() + payloadLength;
    // Payload完整读取后，下次该继续读header了
    readingHeaders = true;
    // 根据FrameType读取Payload，因为不同类型的Payload格式不一样
    switch (frameType) {
        case DATA:
            readDataFrame(ctx, in, payloadEndIndex, listener);
            break;
        case HEADERS:
            readHeadersFrame(ctx, in, payloadEndIndex, listener);
            break;
        case PRIORITY:
            readPriorityFrame(ctx, in, listener);
            break;
        case RST_STREAM:
            readRstStreamFrame(ctx, in, listener);
            break;
        case SETTINGS:
            readSettingsFrame(ctx, in, listener);
            break;
        case PUSH_PROMISE:
            readPushPromiseFrame(ctx, in, payloadEndIndex, listener);
            break;
        case PING:
            readPingFrame(ctx, in.readLong(), listener);
            break;
        case GO_AWAY:
            readGoAwayFrame(ctx, in, payloadEndIndex, listener);
            break;
        case WINDOW_UPDATE:
            readWindowUpdateFrame(ctx, in, listener);
            break;
        case CONTINUATION:
            readContinuationFrame(in, payloadEndIndex, listener);
            break;
        default:
            readUnknownFrame(ctx, in, payloadEndIndex, listener);
            break;
    }
    in.readerIndex(payloadEndIndex);
}

篇幅原因，就不每个Frame的读取都写了，只以最常用的HEADERS和DATAFrame举例。

HeadersFrame

HEADERSFrame的Payload格式如下：

HEADERS Frame {
  [Pad Length (8)],
  [Exclusive (1)],
  [Stream Dependency (31)],
  [Weight (8)],
  Field Block Fragment (..),
  Padding (..2040),
}

DefaultHttp2FrameReader#readHeadersFrame()：读取HEADERSFrame。

private void readHeadersFrame(final ChannelHandlerContext ctx, ByteBuf payload, int payloadEndIndex,
        Http2FrameListener listener) throws Http2Exception {
    final int headersStreamId = streamId;
    final Http2Flags headersFlags = flags;
    // 填充字节长度 没有则为0
    final int padding = readPadding(payload);
    verifyPadding(padding);// 校验
    // 有优先级的处理逻辑  先忽略。。。
    if (flags.priorityPresent()) {
        long word1 = payload.readUnsignedInt();
        final boolean exclusive = (word1 & 0x80000000L) != 0;
        final int streamDependency = (int) (word1 & 0x7FFFFFFFL);
        if (streamDependency == streamId) {
            throw streamError(streamId, PROTOCOL_ERROR, "A stream cannot depend on itself.");
        }
        final short weight = (short) (payload.readUnsignedByte() + 1);
        final int lenToRead = lengthWithoutTrailingPadding(payloadEndIndex - payload.readerIndex(), padding);

        // Create a handler that invokes the listener when the header block is complete.
        headersContinuation = new HeadersContinuation() {
            @Override
            public int getStreamId() {
                return headersStreamId;
            }

            @Override
            public void processFragment(boolean endOfHeaders, ByteBuf fragment, int len,
                    Http2FrameListener listener) throws Http2Exception {
                final HeadersBlockBuilder hdrBlockBuilder = headersBlockBuilder();
                hdrBlockBuilder.addFragment(fragment, len, ctx.alloc(), endOfHeaders);
                if (endOfHeaders) {
                    listener.onHeadersRead(ctx, headersStreamId, hdrBlockBuilder.headers(), streamDependency,
                            weight, exclusive, padding, headersFlags.endOfStream());
                }
            }
        };
        headersContinuation.processFragment(flags.endOfHeaders(), payload, lenToRead, listener);
        resetHeadersContinuationIfEnd(flags.endOfHeaders());
        return;
    }
    // 没有优先级的处理逻辑 重点看
    headersContinuation = new HeadersContinuation() {
        @Override
        public int getStreamId() {
            return headersStreamId;
        }

        @Override
        public void processFragment(boolean endOfHeaders, ByteBuf fragment, int len,
                Http2FrameListener listener) throws Http2Exception {
            final HeadersBlockBuilder hdrBlockBuilder = headersBlockBuilder();
            // 有效数据写入headerBlock
            hdrBlockBuilder.addFragment(fragment, len, ctx.alloc(), endOfHeaders);
            if (endOfHeaders) {
                // 解析headers，触发监听
                listener.onHeadersRead(ctx, headersStreamId, hdrBlockBuilder.headers(), padding,
                                headersFlags.endOfStream());
            }
        }
    };
    // 去除Padding后的长度
    int len = lengthWithoutTrailingPadding(payloadEndIndex - payload.readerIndex(), padding);
    // 处理Headers
    headersContinuation.processFragment(flags.endOfHeaders(), payload, len, listener);
    resetHeadersContinuationIfEnd(flags.endOfHeaders());
}

HTTP2头部使用HPACK压缩了，所以headers的具体解析工作交给了HpackDecoder#decode()，比较复杂，涉及到HPACK静态表、动态表和哈夫曼编码，细节请看笔者的另一篇文章。

DataFrame

DATAFrame的Payload格式如下，很简单，由Data和可选的Padding组成。

DATA Frame {
  [Pad Length (8)],
  Data (..),
  Padding (..2040),
}

DefaultHttp2FrameReader#readDataFrame()：读取DATAFrame并触发监听器，因为结构简单，解析代码很少。

private void readDataFrame(ChannelHandlerContext ctx, ByteBuf payload, int payloadEndIndex,
        Http2FrameListener listener) throws Http2Exception {
    // 读取填充字节长度 没有则为0
    int padding = readPadding(payload);
    verifyPadding(padding);// 校验
    // 去掉填充字节的真实数据长度
    int dataLength = lengthWithoutTrailingPadding(payloadEndIndex - payload.readerIndex(), padding);
    // 从输入缓冲区读取一个切片，触发监听器
    ByteBuf data = payload.readSlice(dataLength);
    listener.onDataRead(ctx, streamId, data, padding, flags.endOfStream());
}

Http2FrameWriter

Netty还提供了io.netty.handler.codec.http2.Http2FrameWriter接口将Frame对象编码为字节序列写入到Channel，让开发者不用关心底层细节。清楚了解码的逻辑，再看编码的逻辑就很简单了。
Http2FrameWriter针对不同类型的Frame提供了对应的方法：

public interface Http2FrameWriter extends Http2DataWriter, Closeable {

ChannelFuture writeHeaders(ChannelHandlerContext ctx, int streamId, Http2Headers headers,
                           int padding, boolean endStream, ChannelPromise promise);
    
ChannelFuture writePriority(ChannelHandlerContext ctx, int streamId, int streamDependency,
        short weight, boolean exclusive, ChannelPromise promise);

ChannelFuture writeRstStream(ChannelHandlerContext ctx, int streamId, long errorCode,
        ChannelPromise promise);

ChannelFuture writeSettings(ChannelHandlerContext ctx, Http2Settings settings,
        ChannelPromise promise);

ChannelFuture writeSettingsAck(ChannelHandlerContext ctx, ChannelPromise promise);

ChannelFuture writePing(ChannelHandlerContext ctx, boolean ack, long data,
        ChannelPromise promise);
。。。。。。
}

DefaultHttp2FrameWriter#writeHeadersInternal()以写入HEADERSFrame为例：

private ChannelFuture writeHeadersInternal(ChannelHandlerContext ctx,
        int streamId, Http2Headers headers, int padding, boolean endStream,
        boolean hasPriority, int streamDependency, short weight, boolean exclusive, ChannelPromise promise) {
    ByteBuf headerBlock = null;
    SimpleChannelPromiseAggregator promiseAggregator =
            new SimpleChannelPromiseAggregator(promise, ctx.channel(), ctx.executor());
    try {
        // 校验StreamID
        verifyStreamId(streamId, STREAM_ID);
        if (hasPriority) {// 有优先级的情况 暂不考虑
            verifyStreamOrConnectionId(streamDependency, STREAM_DEPENDENCY);
            verifyPadding(padding);
            verifyWeight(weight);
        }

        // 申请一个ByteBuf 交给HpackEncoder对headers进行编码
        headerBlock = ctx.alloc().buffer();
        headersEncoder.encodeHeaders(streamId, headers, headerBlock);
        // 生成Flags
        Http2Flags flags =
                new Http2Flags().endOfStream(endStream).priorityPresent(hasPriority).paddingPresent(padding > 0);

        // Read the first fragment (possibly everything).
        int nonFragmentBytes = padding + flags.getNumPriorityBytes();
        int maxFragmentLength = maxFrameSize - nonFragmentBytes;
        ByteBuf fragment = headerBlock.readRetainedSlice(min(headerBlock.readableBytes(), maxFragmentLength));

        // Set the end of headers flag for the first frame.
        flags.endOfHeaders(!headerBlock.isReadable());
        // 按照格式写入Payload
        int payloadLength = fragment.readableBytes() + nonFragmentBytes;
        ByteBuf buf = ctx.alloc().buffer(HEADERS_FRAME_HEADER_LENGTH);
        writeFrameHeaderInternal(buf, payloadLength, HEADERS, flags, streamId);
        writePaddingLength(buf, padding);

        if (hasPriority) {
            buf.writeInt(exclusive ? (int) (0x80000000L | streamDependency) : streamDependency);

            // Adjust the weight so that it fits into a single byte on the wire.
            buf.writeByte(weight - 1);
        }
        ctx.write(buf, promiseAggregator.newPromise());

        // Write the first fragment.
        ctx.write(fragment, promiseAggregator.newPromise());

        // Write out the padding, if any.
        if (paddingBytes(padding) > 0) {
            ctx.write(ZERO_BUFFER.slice(0, paddingBytes(padding)), promiseAggregator.newPromise());
        }
        if (!flags.endOfHeaders()) {
            // 一个Frame无法装在所有headers，后面要跟上ContinuationFrame继续写
            writeContinuationFrames(ctx, streamId, headerBlock, promiseAggregator);
        }
    } catch (Http2Exception e) {
        promiseAggregator.setFailure(e);
    } catch (Throwable t) {
        promiseAggregator.setFailure(t);
        promiseAggregator.doneAllocatingPromises();
        PlatformDependent.throwException(t);
    } finally {
        if (headerBlock != null) {
            headerBlock.release();
        }
    }
    return promiseAggregator.doneAllocatingPromises();
}