Bootstrap

在springboot中操作redis的stream

本文将介绍springboot下监听redis的stream、创建消费组、删除消费组、以及pending队列监控和消息ack和删除

1 基础设施

导入依赖

 <dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-data-redis</artifactId>
  </dependency>
  <dependency>
	<groupId>org.apache.commons</groupId>
	<artifactId>commons-pool2</artifactId>
</dependency>

配置文件

spring:
  redis.:
    host: 192.168.0.3
    port: 6379
    database: 0
    timeout: 15000
    lettuce:
      pool:
        max-idle: 50 # 连接池中的最大空闲连接
        min-idle: 10 # 连接池中的最小空闲连接
        max-active: 300 # 连接池的最大数据库连接数
        max-wait: -1 #连接池最大阻塞等待时间

redis配置类

@Configuration
public class MyRedisConfig {

    @Bean
    public RedisTemplate<String, Object> redisTemplate(RedisConnectionFactory connectionFactory) {
        RedisTemplate<String, Object> redisTemplate = new RedisTemplate<>();
        redisTemplate.setConnectionFactory(connectionFactory);
        redisTemplate.setKeySerializer(new StringRedisSerializer());
        redisTemplate.setValueSerializer(new StringRedisSerializer());
        redisTemplate.setHashValueSerializer(RedisSerializer.string());
        // hash的key也采用String的序列化方式
        redisTemplate.setHashKeySerializer(new StringRedisSerializer());
        Jackson2JsonRedisSerializer<Object> jackson2JsonRedisSerializer = new Jackson2JsonRedisSerializer<>(Object.class);
        ObjectMapper objectMapper = new ObjectMapper();
        objectMapper.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.ANY);
        jackson2JsonRedisSerializer.setObjectMapper(objectMapper);
        objectMapper.activateDefaultTyping(objectMapper.getPolymorphicTypeValidator(), ObjectMapper.DefaultTyping.NON_FINAL, JsonTypeInfo.As.PROPERTY);
        jackson2JsonRedisSerializer.setObjectMapper(objectMapper);
        redisTemplate.setHashValueSerializer(jackson2JsonRedisSerializer);

        return redisTemplate;
    }

}

2 封装消息和消息ID的对象

消息对象的创建:

使用 org.springframework.data.redis.connection.stream.StreamRecords的静态方法来创建消息实例
一个stream消息有两个内容。可以理解为：一个是key，一个是value。
key和value都可以使用自定义的对象，字节，字符串来定义

ByteRecord rawBytes(Map<byte[], byte[]> raw) 

ByteBufferRecord rawBuffer(Map<ByteBuffer, ByteBuffer> raw) 

StringRecord string(Map<String, String> raw)

<S, K, V> MapRecord<S, K, V> mapBacked(Map<K, V> map)

<S, V> ObjectRecord<S, V> objectBacked(V value)

RecordBuilder<?> newRecord()  // 通过builder方式来创建消息

RecordId 表示消息ID

一条消息的ID是唯一的。并且有2部分组成

// ----------- 读取ID属性的实例方法
// 是否是系统自动生成的
boolean shouldBeAutoGenerated();
// 获取原始的id字符串
String getValue();
// 获取序列号部分
long getSequence();
// 获取时间戳部分
long getTimestamp();

// ----------- 创建ID的静态方法
RecordId of(@Nullable String value)
RecordId of(long millisecondsTime, long sequenceNumber)
RecordId autoGenerate()

3 往Stream推送Map消息

3.1 使用RedisTemplate

    @Autowired
    private RedisTemplate<String, Object> redisTemplate;

    public void send1() {
        HashMap<Object, Object> map = new HashMap<>();
        map.put("xxx", "xxx");
        final MapRecord<String, Object, Object> record = StreamRecords.newRecord()
                .in("mystream")
                .ofMap(map)
                .withId(RecordId.autoGenerate());

		RecordId recordId = redisTemplate.opsForStream().add(record);
		// 是否是自动生成的
		boolean autoGenerated = recordId.shouldBeAutoGenerated();
		// id值
		String value = recordId.getValue();
		// 序列号部分
		long sequence = recordId.getSequence();
		// 时间戳部分
		long timestamp = recordId.getTimestamp();
    }
    
    public void send2() {
        StringRecord record = StreamRecords.string(Collections.singletonMap("xxx", xxx))
                .withStreamKey("mystream")
                .withId(RecordId.autoGenerate());
        RecordId recordId = redisTemplate.opsForStream().add(record);

        // 是否是自动生成的
        boolean autoGenerated = recordId.shouldBeAutoGenerated();
        // id值
        String value = recordId.getValue();
        // 序列号部分
        long sequence = recordId.getSequence();
        // 时间戳部分
        long timestamp = recordId.getTimestamp();
    }

3.2 使用RedisConnection

@Autowired
private RedisConnectionFactory redisConnectionFactory;

public void test () {
	// 创建消息记录, 以及指定stream
	ByteRecord byteRecord = StreamRecords.rawBytes(Collections.singletonMap("name".getBytes(), "KevinBlandy".getBytes())).withStreamKey("mystream".getBytes());
	// 获取连接
	RedisConnection redisConnection = this.redisConnectionFactory.getConnection();
	RecordId recordId = redisConnection.xAdd(byteRecord);
	// 是否是自动生成的
	boolean autoGenerated = recordId.shouldBeAutoGenerated();
	// id值
	String value = recordId.getValue();
	// 序列号部分
	long sequence = recordId.getSequence();
	// 时间戳部分
	long timestamp = recordId.getTimestamp();
}

4 往Stream推送对象消息

	@Data
	class Book {
	    private String title;
	    private String author;
	    public static Book create() {
	        Book book = new Book();
	        book.setTitle("xxx");
	        book.setAuthor("xxx");
	        return book;
	    }
	}
	
	@Autowired
	private RedisTemplate<String, Object> redisTemplate;
	
	public void sendRecord(String streamKey) {
	    Book book = Book.create();
	    log.info("产生一本书的信息:[{}]", book);
	    
	    ObjectRecord<String, Book> record = StreamRecords.newRecord()
	            .in(streamKey)
	            .ofObject(book)
	            .withId(RecordId.autoGenerate());
	    
	    RecordId recordId = redisTemplate.opsForStream()
	            .add(record);
	    
	    log.info("返回的record-id:[{}]", recordId);
	}

5 创建指定消费者组，如果之前存在这个消费者组，则删除

	@Autowired
	private RedisTemplate<String, Object> redisTemplate;
	
	private final AtomicBoolean isCreated = new AtomicBoolean(false);
	
	@PostConstruct
	public void groupInfo() {
	
	    // 发送个心跳,保证stream已经存在
	    HashMap<Object, Object> map = new HashMap<>();
	    map.put("fileBeat","fileBeat...");
	    final MapRecord<String, Object, Object> record = StreamRecords.newRecord()
	            .in(StreamConstant.Document.streamName)
	            .ofMap(map)
	            .withId(RecordId.autoGenerate());
	
	    final StreamOperations<String, Object, Object> stream = redisTemplate.opsForStream();
	    stream.add(record);
	
	    final StreamInfo.XInfoGroups xInfoGroups = stream.groups(StreamConstant.Document.streamName);
	
	    Collection<StreamInfo.XInfoGroup> needDestroyColl = new ArrayList<>();
	
	    xInfoGroups.forEach(xInfoStream -> {
	        if (xInfoStream.groupName().equals(StreamConstant.Document.consumerGroup)) {
	            isCreated.set(true);
	        } else {
	            needDestroyColl.add(xInfoStream);
	        }
	    });
	
	    for (StreamInfo.XInfoGroup xInfoGroup : needDestroyColl) {
	        log.info("destroy consumer group[{}]...", xInfoGroup.groupName());
	        stream.destroyGroup(StreamConstant.Document.streamName,xInfoGroup.groupName());
	    }
	
	    if (isCreated.get()) return;
	
	    log.info("create consumer group[{}]...", StreamConstant.Document.consumerGroup);
	
	    stream.createGroup(StreamConstant.Document.streamName, StreamConstant.Document.consumerGroup);
	
	}

6 消费者组模式消费信息

6.1 阻塞消费

阻塞消费监听MapRecord

注意：

  • MapRecord意思就是结果映射成一个 MapRecord<S, K, V>，S是stream，k是自定义消息的key，V是具体消息，发送时也要发送 MapRecord<S, K, V>
  • 需要指定 .keySerializer(new StringRedisSerializer())
@Configuration
public class RedisStreamConfiguration {

	@Autowired
	private RedisConnectionFactory redisConnectionFactory;
	
	@Autowired
	private StateListener2 stateListener;
	
	@Autowired
	@Qualifier("stream-core-pool")
	private ThreadPoolTaskExecutor executor;
	
	@Bean(initMethod = "start", destroyMethod = "stop")
	public StreamMessageListenerContainer<String, MapRecord<String, String, String>> streamMessageListenerContainer() {
	
	    StreamMessageListenerContainer.StreamMessageListenerContainerOptions<String, MapRecord<String, String, String>>
	            options = StreamMessageListenerContainer.StreamMessageListenerContainerOptions
	            .builder()
	            // 一次最多获取多少条消息
	            .batchSize(5)
	            // 	执行消息轮询的执行器
	            .executor(executor)
	            // 超时时间，设置为0，表示不超时（超时后会抛出异常）
	            .pollTimeout(Duration.ZERO)
	            // 消息消费异常的handler
	            .errorHandler(e-> log.error("发生了异常", e))
	            // 序列化器 或者RedisSerializer.string()
	            .serializer(new StringRedisSerializer())
	            .build();
	
	    StreamMessageListenerContainer<String, MapRecord<String, String, String>> streamMessageListenerContainer = StreamMessageListenerContainer
	            .create(this.redisConnectionFactory, options);
	
	    // 消费组B,手动ack
	    // receiveAutoAck(自动ack)
	    streamMessageListenerContainer.receive(
	            Consumer.from(StreamConstant.Document.consumerGroup, StreamConstant.Document.consumerName),
	            StreamOffset.create(StreamConstant.Document.streamName, ReadOffset.lastConsumed()),
	            stateListener
	    );
	
	    return streamMessageListenerContainer;
	}
}

结合自己业务实现StreamListener，并根据业务进行ack或删除消息

@Slf4j
@Configuration
public class StateListener implements StreamListener<String, MapRecord<String,String, String>> {

    @Autowired
    private RedisTemplate<String, Object> redisTemplate;
    
    @Override
    @Async("stream-core-pool")
    public void onMessage(MapRecord<String,String, String> message) {
        StreamOperations<String, Object, Object> opsForStream = redisTemplate.opsForStream();
        RecordId id = message.getId();
        String value = message.getValue().get("xxx");

        if (StrUtil.isBlank(value)){
            log.error("接收到非法信息：{}",message.getValue());
            opsForStream.acknowledge(StreamConstant.Document.streamName, StreamConstant.Document.consumerGroup, id);
            opsForStream.delete(message);
            return;
        }
        // 除去不合法字符
        value = value.replaceAll("\"","");
        log.info("received id:{} uuid:{}", id, value);
        try {
        	// todo 业务
        } catch (Exception e) {
            log.error(e.getMessage());
            e.printStackTrace();
        } finally {
            try {
                final Long acknowledge = opsForStream.acknowledge(
                        StreamConstant.Document.streamName,
                        StreamConstant.Document.consumerGroup,
                        id
                );
                Long delete = 0L;
                if (acknowledge != null && acknowledge == 1L) {
                    delete = opsForStream.delete(message);
                }
                log.info("acknowledge:{} delete:{}",acknowledge, delete);
            } catch (Exception e) {
                log.error(e.getMessage());
                e.printStackTrace();
            }
        }
    }
}

阻塞消费监听ObjectRecord

注意：

  • ObjectRecord意思就是结果映射成一个对象，发送时也要发送对象
  • 需要指定 .keySerializer(new StringRedisSerializer())
  • 使用ObjectRecord接收对象时，指定.objectMapper(new ObjectHashMapper())
@Configuration
public class RedisStreamConfiguration {

    @Data
    public static class Book {
        private String title;
        private String author;
        public static Book create() {
            Book book = new Book();
            book.setTitle("xxx");
            book.setAuthor("xxx");
            return book;
        }
    }

    @Autowired
    private RedisConnectionFactory redisConnectionFactory;

    @Autowired
    private StateListener stateListener;

    @Autowired
    @Qualifier("stream-core-pool")
    private ThreadPoolTaskExecutor executor;

    @Bean(initMethod = "start", destroyMethod = "stop")
    public StreamMessageListenerContainer<String, ObjectRecord<String, Book>> streamMessageListenerContainer() {

        StreamMessageListenerContainer.StreamMessageListenerContainerOptions<String, ObjectRecord<String, Book>>
                options = StreamMessageListenerContainer.StreamMessageListenerContainerOptions
                .builder()
                // 一次最多获取多少条消息
                .batchSize(5)
                // 	执行消息轮询的执行器
                .executor(executor)
                // 超时时间，设置为0，表示不超时（超时后会抛出异常）
                .pollTimeout(Duration.ZERO)
                // 消息消费异常的handler
                .errorHandler(new CustomErrorHandler())
                // 序列化器 或者RedisSerializer.string()
                .serializer(new StringRedisSerializer())
                .keySerializer(new StringRedisSerializer())
                .hashKeySerializer(new StringRedisSerializer())
                .hashValueSerializer(new StringRedisSerializer())
                .objectMapper(new ObjectHashMapper())
                .targetType(Book.class)
                .build();

        StreamMessageListenerContainer<String, ObjectRecord<String, Book>> streamMessageListenerContainer =
                StreamMessageListenerContainer.create(redisConnectionFactory, options);

        // 消费组B,手动ack
        // receiveAutoAck(自动ack)
        streamMessageListenerContainer.receive(
                Consumer.from(StreamConstant.Document.consumerGroup, StreamConstant.Document.consumerName),
                StreamOffset.create(StreamConstant.Document.streamName, ReadOffset.lastConsumed()),
                stateListener
        );

        return streamMessageListenerContainer;
    }
}

结合自己业务实现StreamListener，并根据业务进行ack或删除消息

@Slf4j
@Configuration
public class StateListener implements StreamListener<String, ObjectRecord<String, Book>> {

    @Autowired
    private RedisTemplate<String, Object> redisTemplate;
    
    @Override
    @Async("stream-core-pool")
    public void onMessage(ObjectRecord<String, Book> message) {
      // 同上
    }
}

6.2 非阻塞消费

主要是通过StreamOperations 或者是 RedicConnection 的消费API来进行消息的随机消费

从RedisTemplate中获取到StreamOperations

StreamOperations<String, Object, Object> opsForStream = redisTemplate.opsForStream();

StreamOperations 的读取 API:

// 随机范围读取
<V> List<ObjectRecord<K, V>> range(Class<V> targetType, K key, Range<String> range)
<V> List<ObjectRecord<K, V>> range(Class<V> targetType, K key, Range<String> range, Limit limit)


// 根据消息ID或者偏移量读取
List<MapRecord<K, HK, HV>> read(StreamOffset<K>... streams)
<V> List<ObjectRecord<K, V>> read(Class<V> targetType, StreamOffset<K>... streams)
List<MapRecord<K, HK, HV>> read(StreamReadOptions readOptions, StreamOffset<K>... streams)
<V> List<ObjectRecord<K, V>> read(Class<V> targetType, StreamReadOptions readOptions, StreamOffset<K>... streams)
List<MapRecord<K, HK, HV>> read(Consumer consumer, StreamOffset<K>... streams)
<V> List<ObjectRecord<K, V>> read(Class<V> targetType, Consumer consumer, StreamOffset<K>... streams)
List<MapRecord<K, HK, HV>> read(Consumer consumer, StreamReadOptions readOptions, StreamOffset<K>... streams)
List<ObjectRecord<K, V>> read(Class<V> targetType, Consumer consumer, StreamReadOptions readOptions, StreamOffset<K>... streams)

// 随机逆向范围读取
List<MapRecord<K, HK, HV>> reverseRange(K key, Range<String> range)
List<MapRecord<K, HK, HV>> reverseRange(K key, Range<String> range, Limit limit)
<V> List<ObjectRecord<K, V>> reverseRange(Class<V> targetType, K key, Range<String> range)
<V> List<ObjectRecord<K, V>> reverseRange(Class<V> targetType, K key, Range<String> range, Limit limit)

// 消费者信息
XInfoConsumers consumers(K key, String group);
// 消费者信息
XInfoGroups groups(K key);
// stream信息
XInfoStream info(K key);

// 获取消费组，消费者中未确认的消息
PendingMessagesSummary pending(K key, String group);
PendingMessages pending(K key, Consumer consumer)
PendingMessages pending(K key, String group, Range<?> range, long count)
PendingMessages pending(K key, String group, Range<?> range, long count)

7 监听未ACK的消息并处理

注意：使用此方式需要给redisTemplate的hashValue序列化方式配置为String

@Component
@Slf4j
public class NonAckHandle {

    @Autowired
    @Qualifier("myStringRedisTemplate")
    private RedisTemplate<String, Object> redisTemplate;

    //    @Scheduled(cron = "0 0/1 * * * ?")
    @Scheduled(cron = "0/10 0/1 * * * ?")
    public void doMonitor2() {
 		 // 一次取10条未ack的消息
        StreamOperations<String, String, String> stream = redisTemplate.opsForStream();

        //  XPENDING doc-state-stream doc-state-stream-group - + 10
        PendingMessages pendingMessages = stream.pending(
                StreamConstant.Document.streamName,
                StreamConstant.Document.consumerGroup,
                Range.of(Range.Bound.inclusive("-"),Range.Bound.inclusive("+")),
                10
        );

        // 如果空
        if (pendingMessages.isEmpty()) {
            return;
        }
        for (PendingMessage next : pendingMessages) {
            Consumer consumer = next.getConsumer();
            RecordId recordId = next.getId();
            log.warn("consumer={} RecordId={}", consumer, recordId);


            // XRANGE doc-state-stream 1676974558423 1676974558423 count 1
            final List<MapRecord<String, String, String>> res = stream.range(
                    StreamConstant.Document.streamName,
                    Range.of(Range.Bound.inclusive(recordId.getValue()),Range.Bound.inclusive(recordId.getValue())),
                    RedisZSetCommands.Limit.limit().count(1)
            );

            //
//            List<MapRecord<String, String, String>> res2 = stream.read(
//                    StreamReadOptions.empty().count(1),
//                    StreamOffset.create(StreamConstant.Document.streamName, ReadOffset.from(recordId))
//            );

            res.forEach(ele -> this.acknowledge(ele, stream, recordId));

        }

    }

    public void acknowledge(MapRecord<String, String, String> message, StreamOperations<String, String, String> opsForStream, RecordId recordId) {
        String value = message.getValue().get("fileUuid");
        if (StrUtil.isBlank(value)){
            log.warn("接收到非法信息：{}", message.getValue());
            ackAndDel(message, opsForStream, recordId);
            return;
        }
        // 除去不合法字符
        value = value.replaceAll("\"","");
        log.info("received id:{} uuid:{}", recordId, value);
        try {
            // todo 业务
        } catch (Exception e) {
            log.error(e.getMessage());
            e.printStackTrace();
        } finally {
            try {
                ackAndDel(message, opsForStream, recordId);
            } catch (Exception e) {
                log.error(e.getMessage());
                e.printStackTrace();
            }
        }
    }

    private void ackAndDel(MapRecord<String, String, String> message, StreamOperations<String, String, String> opsForStream, RecordId recordId) {
        final Long acknowledge = opsForStream.acknowledge(
                StreamConstant.Document.streamName,
                StreamConstant.Document.consumerGroup,
                recordId
        );
        Long delete = 0L;
        if (acknowledge != null && acknowledge == 1L) {
            delete = opsForStream.delete(message);
        }
        log.info("acknowledge:{} delete:{}",acknowledge, delete);
    }
}

悦读

道可道，非常道；名可名，非常名。 无名，天地之始，有名，万物之母。 故常无欲，以观其妙，常有欲，以观其徼。 此两者，同出而异名，同谓之玄，玄之又玄，众妙之门。

最新收录
c# 抽象方法 虚函数 使用场景
AIGC专栏11——EasyAnimateV2结构详解与Lora训练 最大支持768x768 144帧视频生成
Java对象转换成XML
WPS JS宏单元格双击打勾
Ciallo～(∠・ω・ )⌒☆第二篇 Python运算符
网络编程中的TCP和UDP
HTML5期末复习
Hadoop MapReduce编程 API入门系列之多个Job迭代式MapReduce运行（十二）
Java——进制转换
ant design of vue3 文件上传(Upload),自定义customRequest
;