【笔记】单机100万QPS的ID生成器

服务器准备

程序准备

pom文件

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>org.example</groupId>
    <artifactId>NettyTest</artifactId>
    <version>1.0-SNAPSHOT</version>

    <properties>
        <maven.compiler.source>8</maven.compiler.source>
        <maven.compiler.target>8</maven.compiler.target>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
    </properties>

    <dependencies>
        <!-- https://mvnrepository.com/artifact/io.netty/netty-all -->
        <dependency>
            <groupId>io.netty</groupId>
            <artifactId>netty-all</artifactId>
            <version>4.1.112.Final</version>
        </dependency>

    </dependencies>

    <build>
        <plugins>
            <!-- 主动声明插件 -->
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-jar-plugin</artifactId>
                <version>3.1.2</version>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-dependency-plugin</artifactId>
                <executions>
                    <execution>
                        <!-- 绑定生命周期 -->
                        <phase>package</phase>
                        <goals>
                            <goal>copy-dependencies</goal>
                        </goals>
                        <!-- 设置依赖的存放路径 -->
                        <configuration>
                            <outputDirectory>${project.build.directory}/lib</outputDirectory>
                        </configuration>
                    </execution>
                </executions>
            </plugin>

        </plugins>
    </build>
</project>

logback.xml

<configuration debug="false">
  <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
    <encoder>
      <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
    </encoder>
  </appender>

  <root level="${logLevel:-info}">
    <appender-ref ref="STDOUT" />
  </root>
</configuration>

服务端代码

org.example.counter.server.Main

package org.example.counter.server;

import io.netty.bootstrap.ServerBootstrap;
import io.netty.buffer.AbstractByteBufAllocator;
import io.netty.buffer.ByteBufAllocator;
import io.netty.buffer.PooledByteBufAllocator;
import io.netty.buffer.UnpooledByteBufAllocator;
import io.netty.channel.*;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioServerSocketChannel;
import io.netty.handler.codec.LengthFieldBasedFrameDecoder;
import io.netty.handler.codec.LengthFieldPrepender;
import io.netty.handler.codec.LineBasedFrameDecoder;
import io.netty.handler.codec.string.LineEncoder;
import io.netty.handler.codec.string.LineSeparator;
import io.netty.handler.codec.string.StringDecoder;
import io.netty.handler.codec.string.StringEncoder;
import io.netty.handler.logging.LogLevel;
import io.netty.handler.logging.LoggingHandler;
import io.netty.util.AttributeKey;
import io.netty.util.CharsetUtil;
import org.example.counter.client.CounterClientRequestHandler;
import org.example.counter.client.CounterClientResponeHandler;

import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import static java.lang.System.out;

// 按两次 Shift 打开“随处搜索”对话框并输入 `show whitespaces`，
// 然后按 Enter 键。现在，您可以在代码中看到空格字符。
public class Main {

    public static void main(String[] args) {

        EventLoopGroup bossGroup = new NioEventLoopGroup(1);
        EventLoopGroup workerGroup = new NioEventLoopGroup();

        CounterServerHandler serverHandler = new CounterServerHandler();


        new ServerBootstrap()
                .group(bossGroup,workerGroup)
                .channel(NioServerSocketChannel.class)
//                .option(ChannelOption.SO_BACKLOG, 1024000)
                .handler(new LoggingHandler(LogLevel.INFO))
                .childOption(ChannelOption.CONNECT_TIMEOUT_MILLIS, 10000)
                .childOption(ChannelOption.TCP_NODELAY, false)
                .childOption(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT)
                .childHandler(new ChannelInitializer<SocketChannel>() {
                    @Override
                    public void initChannel(SocketChannel ch) throws Exception {
                        ChannelPipeline p = ch.pipeline();

                        ChannelPipeline pipeline = ch.pipeline();

                        // 解决粘包，通过长度字段来拆包和封包
                        // Decoders
                        pipeline.addLast("frameDecoder", new LineBasedFrameDecoder(8192));
                        pipeline.addLast("stringDecoder", new StringDecoder(CharsetUtil.UTF_8));

                        pipeline.addLast(serverHandler );

                        // Encoder
                        pipeline.addLast("lineEncoder", new LineEncoder(LineSeparator.UNIX, CharsetUtil.UTF_8));

                    }
                }).bind(7070);

    }
}

org.example.counter.server.CounterServerHandler

/*
 * Copyright 2012 The Netty Project
 *
 * The Netty Project licenses this file to you under the Apache License,
 * version 2.0 (the "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at:
 *
 *   https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations
 * under the License.
 */
package org.example.counter.server;

import io.netty.channel.ChannelHandler.Sharable;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;

import java.util.concurrent.atomic.AtomicInteger;

/**
 * Handler implementation for the echo server.
 */
@Sharable
public class CounterServerHandler extends ChannelInboundHandlerAdapter {

    private final AtomicInteger counter = new AtomicInteger(0);

    public void reset() {
        counter.set(0);
    }

    public String getCount() {
        return String.valueOf(counter.get());
    }

    public String incrementAndGet() {
        return String.valueOf(counter.incrementAndGet());
    }


    @Override
    public void channelRead(ChannelHandlerContext ctx, Object msg) {

//        System.out.println("receive msg: " + msg);

        if(msg instanceof String)
        {
            if(msg.equals("reset"))
            {
                reset();

                ctx.channel().write("reset success");
            }
            else if(msg.equals("get"))
            {
                ctx.channel().write(getCount());
            }
            else if(msg.equals("incr"))
            {
                ctx.channel().write( incrementAndGet() );
            }
            else {
                ctx.channel().write("unknow command");
            }
        }
        else {
            ctx.channel().write("unknow command");
        }
    }

    @Override
    public void channelReadComplete(ChannelHandlerContext ctx) {
        ctx.flush();
    }

    @Override
    public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
        // Close the connection when an exception is raised.
        cause.printStackTrace();
        ctx.close();
    }
}

客户端代码

org.example.counter.client.Main

package org.example.counter.client;

import io.netty.bootstrap.Bootstrap;
import io.netty.buffer.*;
import io.netty.channel.*;
import io.netty.channel.nio.NioEventLoopGroup;
import io.netty.channel.socket.SocketChannel;
import io.netty.channel.socket.nio.NioSocketChannel;
import io.netty.handler.codec.LengthFieldBasedFrameDecoder;
import io.netty.handler.codec.LengthFieldPrepender;
import io.netty.handler.codec.LineBasedFrameDecoder;
import io.netty.handler.codec.string.LineEncoder;
import io.netty.handler.codec.string.LineSeparator;
import io.netty.handler.codec.string.StringDecoder;
import io.netty.handler.codec.string.StringEncoder;
import io.netty.util.AttributeKey;
import io.netty.util.CharsetUtil;

import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.LinkedTransferQueue;
import java.util.concurrent.atomic.AtomicLong;

// 按两次 Shift 打开“随处搜索”对话框并输入 `show whitespaces`，
// 然后按 Enter 键。现在，您可以在代码中看到空格字符。
public class Main {


    private static long LIMIT = Runtime.getRuntime().freeMemory() / 1024 / 2;

    static
    {
        System.out.println("背压上限: " + LIMIT );
    }

    public static void main(String[] args) throws InterruptedException, ExecutionException {

        int requests = args.length > 0 ?  Integer.parseInt(args[0]) : 1;

        int connections = args.length > 1 ? Integer.parseInt(args[1]) : 1;

        int threads = args.length > 2 ? Integer.parseInt(args[2]) : 1;

        NioEventLoopGroup group = new NioEventLoopGroup(threads);
        Bootstrap bootstrap = new Bootstrap()
                .group(group)
                .channel(NioSocketChannel.class)
                .option(ChannelOption.TCP_NODELAY, false)
                .option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT)
                .handler(new ChannelInitializer<SocketChannel>() {
                    @Override
                    protected void initChannel(SocketChannel ch) throws Exception {

                        ch.attr(CounterClientRequestHandler.getQueueKey()).set(new LinkedTransferQueue<>());

                        ChannelPipeline pipeline = ch.pipeline();

                        // 解决粘包，通过长度字段来拆包和封包
                        // Decoders
                        pipeline.addLast("frameDecoder", new LineBasedFrameDecoder(8192));
                        pipeline.addLast("stringDecoder", new StringDecoder(CharsetUtil.UTF_8));

                        pipeline.addLast(new CounterClientResponeHandler() );

                        // Encoder
                        pipeline.addLast("lineEncoder", new LineEncoder(LineSeparator.UNIX, CharsetUtil.UTF_8));
                        pipeline.addLast(new CounterClientRequestHandler() );

                    }
                });

        ChannelFuture[] channels = new ChannelFuture[connections];


        for (int i = 0; i < connections; i++) {
//            channels[i] =  bootstrap.connect("127.0.0.1", 7070).sync();
            channels[i] =  bootstrap.connect("192.168.253.201", 7070).sync();
        }

        System.out.println("建立了 " + connections + " 个连接");

        {
            for (int i = 0; i < connections; i++) {
                CompletableFuture<String> responseFuture = new CompletableFuture<>();

                channels[i].channel().writeAndFlush(new Request("reset", responseFuture));

                System.out.println(responseFuture.get());
            }

        }

        System.out.println("验证了 " + connections + " 个连接");


        CountDownLatch latch = new CountDownLatch(requests);
        AtomicLong count = new AtomicLong();
        long start = System.currentTimeMillis();
        for (int i = 0; i < requests; i++) {
            CompletableFuture<String> responseFuture = new CompletableFuture<>();
            channels[ i % connections ].channel().writeAndFlush(new Request("incr",responseFuture));
            count.incrementAndGet();
            responseFuture.whenComplete((s, throwable) -> {

                latch.countDown();
                count.decrementAndGet();
                if (throwable != null)
                {
                    throwable.printStackTrace();
                }else
                {
                    if( latch.getCount() % 1000000 == 0 ) {
                        System.out.println("response " + s);
                    }
                }
            });

            while ( count.get() > LIMIT )
            {
                Thread.yield();
            }

        }

        latch.await();
        long end = System.currentTimeMillis();

        long delta = end -start;

        for (int i = 0; i < connections; i++) {
            channels[i].channel().close().sync();
        }

        System.out.println("关闭了 " + connections + " 个连接");
        System.out.println();


        System.out.println("请求 " + requests + " 个");
        System.out.println("耗时 " + delta + " ms");
        System.out.println("QPS " + requests * 1000L / delta + " 次/秒");



        System.exit(0);
    }
}

org.example.counter.client.Request

package org.example.counter.client;

import java.util.concurrent.CompletableFuture;

public class Request {

    public final String msg;
    public final CompletableFuture<String> future;

    public Request(String msg, CompletableFuture<String> future) {
        this.msg = msg;
        this.future = future;
    }
}

org.example.counter.client.CounterClientRequestHandler

/*
 * Copyright 2012 The Netty Project
 *
 * The Netty Project licenses this file to you under the Apache License,
 * version 2.0 (the "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at:
 *
 *   https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations
 * under the License.
 */
package org.example.counter.client;

import io.netty.channel.ChannelHandler.Sharable;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import io.netty.channel.ChannelOutboundHandlerAdapter;
import io.netty.channel.ChannelPromise;
import io.netty.util.Attribute;
import io.netty.util.AttributeKey;

import java.util.Queue;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.LinkedTransferQueue;

/**
 * Handler implementation for the echo server.
 */
@Sharable
public class CounterClientRequestHandler extends ChannelOutboundHandlerAdapter {

    private static final AttributeKey<LinkedTransferQueue<CompletableFuture<String>>>
            queueKey = AttributeKey.valueOf("queue");

    public static AttributeKey<LinkedTransferQueue<CompletableFuture<String>>>  getQueueKey()
    {
        return queueKey;
    }

    @Override
    public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception {
        Attribute<LinkedTransferQueue<CompletableFuture<String>>> queue =ctx.channel().attr(queueKey);

        if(msg instanceof Request )
        {
            queue.get().put(((Request) msg).future);
            ctx.write(((Request) msg).msg, promise);

        }

    }

    @Override
    public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
        // Close the connection when an exception is raised.
        cause.printStackTrace();
        ctx.close();
    }
}

org.example.counter.client.CounterClientResponeHandler

/*
 * Copyright 2012 The Netty Project
 *
 * The Netty Project licenses this file to you under the Apache License,
 * version 2.0 (the "License"); you may not use this file except in compliance
 * with the License. You may obtain a copy of the License at:
 *
 *   https://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations
 * under the License.
 */
package org.example.counter.client;

import io.netty.channel.ChannelHandler.Sharable;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import io.netty.util.Attribute;
import io.netty.util.AttributeKey;

import java.util.Queue;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.LinkedTransferQueue;

/**
 * Handler implementation for the echo server.
 */
@Sharable
public class CounterClientResponeHandler extends ChannelInboundHandlerAdapter {


    @Override
    public void channelRead(ChannelHandlerContext ctx, Object msg) {

        Attribute<Object> future = ctx.channel().attr(AttributeKey.valueOf("queue"));

        Object object = future.get();

        if( object instanceof LinkedTransferQueue)
        {
            Object o = ((LinkedTransferQueue<?>) object).poll();
            if( o instanceof CompletableFuture )
            {
                ((CompletableFuture)o).complete(msg);
            }
        }
    }

    @Override
    public void channelReadComplete(ChannelHandlerContext ctx) {
        ctx.flush();
    }

    @Override
    public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
        // Close the connection when an exception is raised.
        cause.printStackTrace();
        ctx.close();
    }
}

压测

服务端启动命令

nohup java -server -Djava.net.preferIPv4Stack=true -Dio.netty.leakDetectionLevel=advanced -Xmx3048m -Xms3048m -Xmn2848m -cp NettyTest-1.0-SNAPSHOT.jar:lib/* org.example.counter.server.Main &

客户端运行参数

压测结果