bigdata16
com.shujia
1.0-SNAPSHOT
4.0.0
Flink
1.0
8
8
1.11.2
2.11
2.11.12
2.12.1
org.apache.flink
flink-walkthrough-common_${scala.binary.version}
${flink.version}
org.apache.flink
flink-streaming-scala_${scala.binary.version}
${flink.version}
org.apache.flink
flink-clients_${scala.binary.version}
${flink.version}
org.apache.logging.log4j
log4j-slf4j-impl
${log4j.version}
org.apache.logging.log4j
log4j-api
${log4j.version}
org.apache.logging.log4j
log4j-core
${log4j.version}
org.apache.maven.plugins
maven-compiler-plugin
3.1
1.8
1.8
org.scala-tools
maven-scala-plugin
2.15.2
compile
testCompile
package com.shujia.core
import org.apache.flink.streaming.api.scala._
object Demo01WordCount {
def main(args: Array[String]): Unit = {
val env: StreamExecutionEnvironment= StreamExecutionEnvironment.getExecutionEnvironment
//默认并行度等于CPU的逻辑核数 相当于是任务的一个并行度
env.setParallelism(2)
val linesDS: DataStream[String] = env.socketTextStream("master", 8888)
//对每一条数据进行切分
val wordsDS: DataStream[String] = linesDS
.flatMap(line => line.split(","))
//将每个单词变成 K V 格式
val wordsKVDS: DataStream[(String, Int)] = wordsDS.map(word => (word, 1))
//按照每个单词进行分组
val keyByDS: KeyedStream[(String, Int), String] = wordsKVDS.keyBy(kv => kv._1)
//统计每个单词的数量
val wordCntDS: DataStream[(String, Int)] = keyByDS.sum(1)
//将结果打印
wordCntDS.print()
//启动任务
env.execute("Demo01WordCount")
}
}