添加依赖
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-kafka</artifactId>
<version>${flink.version}</version>
</dependency>
这里将配置文件传入运行环境 args
public class FlinkUtil {
// 环境
public static StreamExecutionEnvironment environment = StreamExecutionEnvironment.getExecutionEnvironment().setParallelism(1);
public static ParameterTool parameterTool;
public static <T> DataStream<T> createFlinkKafkaStream(String[] args,Class<? extends DeserializationSchema<T>> deserialization) throws IOException, InstantiationException, IllegalAccessException {
// 利用参数工具得到用户传入的配置文件
parameterTool = ParameterTool.fromPropertiesFile(args[0]);
// 抽取参数
long checkInterval = parameterTool.getLong("checkpoint.interval",30000L);
String checkPath = parameterTool.get("checkpoint.path");
String checkTopics = parameterTool.get("kafka.input.topics");
Properties properties = parameterTool.getProperties();
List<String> topics = Arrays.asList(checkTopics.split(","));
// 设置环境的检查点和容错机制
environment.enableCheckpointing(checkInterval, CheckpointingMode.EXACTLY_ONCE);
// 设置状态后端
environment.setStateBackend(new FsStateBackend(checkPath));
// 设置外部检查点
environment.getCheckpointConfig().enableExternalizedCheckpoints(CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);
// 创建消费者
FlinkKafkaConsumer<T> consumer = new FlinkKafkaConsumer(topics, deserialization.newInstance(), properties);
// 关闭自动提交检查点???
consumer.setCommitOffsetsOnCheckpoints(false);
// 返回消费者流
return environment.addSource(consumer);
}