一、要搞定一个自定义通信协议其实有三个关键步骤，其他和普通开发流程都一样。

1、首先需要定义一个协议；

2、其次定制一个编码器，便于发送端把自定义的协议给解码成数据流发送到网络上；

3、最后定制一个解码器，便于接收端把收到的数据流再解码成自定协议的数据结构；

下面我们分别纤细介绍这三个数据结构。

首先自定一个协议

@Data
@AllArgsConstructor
public class NettyProtocol {
    /**
     * 0x00 表示请求数据;
     * 0x01 表示响应数据;
     */
    byte type;
    //表示数据部分数据长度，便于在解码时通过这个长度解决拆沾包问题
    int bodyLength;
    //有效数据体
    String body;
}

定制一个编码器

public class NettyProtocolEncoder extends MessageToByteEncoder<NettyProtocol> {
    @Override
    protected void encode(ChannelHandlerContext ctx, NettyProtocol msg, ByteBuf out) throws Exception {
        //将数据结构的标识写入数据流
        out.writeByte(msg.getType());
        //将字符串转字节数组，然后写入数据流
        byte[] body = msg.getBody().getBytes(CharsetUtil.UTF_8);
        msg.setBodyLength(body.length);
        out.writeInt(msg.getBodyLength());
        //将主体数据写入数据流
        out.writeBytes(body);
    }
}

定制一个解码器（理解解码器中state变量使用逻辑是解决粘包拆包问题关键）

public class NettyProtocolDecoder extends ByteToMessageDecoder {
    //注意每次数据写入都会触发一次，解码调用，state作用是为了保存目前正在处理协议头还是数据体的状态，便于在数据不完整时，保存处理状态并等待下次被触发调用
    private enum State {
        Header,
        Body
    }
    private State state = State.Header;
    private int bodyLength;
    private String body;

    @Override
    protected void decode(ChannelHandlerContext ctx, ByteBuf in, List<Object> out) {
        switch (state) {
            case Header:
                if (in.readableBytes() < 5) {
                    System.out.println("Header is not received over!");
                    return;
                }
                in.skipBytes(1);
                bodyLength = in.readInt();
                state = State.Body;
            case Body:
                if (in.readableBytes() < bodyLength) {
                    System.out.println("Body is not received over!");
                    return;
                }
                ByteBuf bodyBytes = in.readBytes(bodyLength);
                body = bodyBytes.toString(CharsetUtil.UTF_8);
                out.add(new NettyProtocol((byte)0x01,bodyLength,body));
                state = State.Header;
        }

    }
}

二、组装服务端代码 

public class NettyServer {
    public static void startServer(String hostName, int port) {
        startServer0(hostName, port);
    }

    private static void startServer0(String hostname, int port) {

        EventLoopGroup bossGroup = new NioEventLoopGroup(1);
        EventLoopGroup workerGroup = new NioEventLoopGroup();

        try {
            SslContext sslCtx = SslContextBuilder.forServer(certChainFile, keyFile).trustManager(rootFile).clientAuth(ClientAuth.REQUIRE).build();
            ServerBootstrap serverBootstrap = new ServerBootstrap();

            serverBootstrap.group(bossGroup, workerGroup)
                    .channel(NioServerSocketChannel.class)
                    .childHandler(new ChannelInitializer<SocketChannel>() {
                                      @Override
                                      protected void initChannel(SocketChannel ch) throws Exception {
                                          //添加编码器
                                          pipeline.addLast(new NettyProtocolEncoder());
                                          //添加解码器
                                          pipeline.addLast(new NettyProtocolDecoder());
                                          //添加处理器
                                          pipeline.addLast(new NettyServerHandler());
                                      }
                                  }

                    );

            ChannelFuture channelFuture = serverBootstrap.bind(hostname, port).sync();
            System.out.println("服务提供方开始提供服务~~");
            channelFuture.channel().closeFuture().sync();

        } catch (Exception e) {
            e.printStackTrace();
        } finally {
            bossGroup.shutdownGracefully();
            workerGroup.shutdownGracefully();
        }
    }
}

三、组装客户端

private static void initClient() {
        try {
        SslContext sslCtx = SslContextBuilder.forClient().keyManager(certChainFile, keyFile).trustManager(rootFile).build();
        //这里是自定义的客户端处理器
        client = new NettyClientHandler();
        //创建EventLoopGroup
        NioEventLoopGroup group = new NioEventLoopGroup();
        Bootstrap bootstrap = new Bootstrap();
        bootstrap.group(group)
                .channel(NioSocketChannel.class)
                .option(ChannelOption.TCP_NODELAY, true)
                .handler(
                        new ChannelInitializer<SocketChannel>() {
                            @Override
                            protected void initChannel(SocketChannel ch) throws Exception {
                                ChannelPipeline pipeline = ch.pipeline();
                                pipeline.addLast(new NettyProtocolEncoder());
                                pipeline.addLast(new NettyProtocolDecoder());
                                pipeline.addLast(client);
                            }
                        }
                );


            bootstrap.connect("127.0.0.1", 7000).sync();
        } catch (Exception e) {
            e.printStackTrace();
        }
    }