Bootstrap

Flink 消费 kafka

添加依赖

<dependency>
  <groupId>org.apache.flink</groupId>
  <artifactId>flink-connector-kafka</artifactId>
  <version>${flink.version}</version>
</dependency>

这里将配置文件传入运行环境 args

public class FlinkUtil {
    // 环境
    public static StreamExecutionEnvironment environment =  StreamExecutionEnvironment.getExecutionEnvironment().setParallelism(1);
    public static ParameterTool parameterTool;

    public static <T> DataStream<T> createFlinkKafkaStream(String[] args,Class<? extends DeserializationSchema<T>> deserialization) throws IOException, InstantiationException, IllegalAccessException {
        // 利用参数工具得到用户传入的配置文件
        parameterTool = ParameterTool.fromPropertiesFile(args[0]);

        // 抽取参数
        long checkInterval = parameterTool.getLong("checkpoint.interval",30000L);
        String checkPath = parameterTool.get("checkpoint.path");
        String checkTopics = parameterTool.get("kafka.input.topics");
        Properties properties = parameterTool.getProperties();
        List<String> topics = Arrays.asList(checkTopics.split(","));

        // 设置环境的检查点和容错机制
        environment.enableCheckpointing(checkInterval, CheckpointingMode.EXACTLY_ONCE);
        // 设置状态后端
        environment.setStateBackend(new FsStateBackend(checkPath));

        // 设置外部检查点
        environment.getCheckpointConfig().enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);

        // 创建消费者
        FlinkKafkaConsumer<T> consumer = new FlinkKafkaConsumer(topics, deserialization.newInstance(), properties);

        // 关闭自动提交检查点???
        consumer.setCommitOffsetsOnCheckpoints(false);

        // 返回消费者流
        return environment.addSource(consumer);
    }

;