Download as PDF, PPTX
![Samza
class WordCountTask extends StreamTask {
override def process(envelope: IncomingMessageEnvelope,
collector: MessageCollector, coordinator: TaskCoordinator) {
val text = envelope.getMessage.asInstanceOf[String]
val counts = text.split(" ")
.foldLeft(Map.empty[String, Int]) {
(count, word) => count + (word -> (count.getOrElse(word, 0) + 1))
}
collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", "wordcount"),
counts))
}](https://image.slidesharecdn.com/sd2016-160512034804/75/Distributed-Real-Time-Stream-Processing-Why-and-How-2-0-39-2048.jpg)
![Samza
class WordCountTask extends StreamTask {
override def process(envelope: IncomingMessageEnvelope,
collector: MessageCollector, coordinator: TaskCoordinator) {
val text = envelope.getMessage.asInstanceOf[String]
val counts = text.split(" ")
.foldLeft(Map.empty[String, Int]) {
(count, word) => count + (word -> (count.getOrElse(word, 0) + 1))
}
collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", "wordcount"),
counts))
}](https://image.slidesharecdn.com/sd2016-160512034804/75/Distributed-Real-Time-Stream-Processing-Why-and-How-2-0-40-2048.jpg)
![class WordCountTask extends StreamTask {
override def process(envelope: IncomingMessageEnvelope,
collector: MessageCollector, coordinator: TaskCoordinator) {
val text = envelope.getMessage.asInstanceOf[String]
val counts = text.split(" ")
.foldLeft(Map.empty[String, Int]) {
(count, word) => count + (word -> (count.getOrElse(word, 0) + 1))
}
collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", "wordcount"),
counts))
}
Samza](https://image.slidesharecdn.com/sd2016-160512034804/75/Distributed-Real-Time-Stream-Processing-Why-and-How-2-0-41-2048.jpg)
![Apex
val input = dag.addOperator("input", new LineReader)
val parser = dag.addOperator("parser", new Parser)
val out = dag.addOperator("console", new ConsoleOutputOperator)
dag.addStream[String]("lines", input.out, parser.in)
dag.addStream[String]("words", parser.out, counter.data)
class Parser extends BaseOperator {
@transient
val out = new DefaultOutputPort[String]()
@transient
val in = new DefaultInputPort[String]()
override def process(t: String): Unit = {
for(w <- t.split(" ")) out.emit(w)
}
}](https://image.slidesharecdn.com/sd2016-160512034804/75/Distributed-Real-Time-Stream-Processing-Why-and-How-2-0-42-2048.jpg)
![Apex
val input = dag.addOperator("input", new LineReader)
val parser = dag.addOperator("parser", new Parser)
val out = dag.addOperator("console", new ConsoleOutputOperator)
dag.addStream[String]("lines", input.out, parser.in)
dag.addStream[String]("words", parser.out, counter.data)
class Parser extends BaseOperator {
@transient
val out = new DefaultOutputPort[String]()
@transient
val in = new DefaultInputPort[String]()
override def process(t: String): Unit = {
for(w <- t.split(" ")) out.emit(w)
}
}](https://image.slidesharecdn.com/sd2016-160512034804/75/Distributed-Real-Time-Stream-Processing-Why-and-How-2-0-43-2048.jpg)
![val input = dag.addOperator("input", new LineReader)
val parser = dag.addOperator("parser", new Parser)
val out = dag.addOperator("console", new ConsoleOutputOperator)
dag.addStream[String]("lines", input.out, parser.in)
dag.addStream[String]("words", parser.out, counter.data)
class Parser extends BaseOperator {
@transient
val out = new DefaultOutputPort[String]()
@transient
val in = new DefaultInputPort[String]()
override def process(t: String): Unit = {
for(w <- t.split(" ")) out.emit(w)
}
}
Apex](https://image.slidesharecdn.com/sd2016-160512034804/75/Distributed-Real-Time-Stream-Processing-Why-and-How-2-0-44-2048.jpg)
![Spark Streaming
// Initial RDD input to updateStateByKey
val initialRDD = ssc.sparkContext.parallelize(List.empty[(String, Int)])
val lines = ...
val words = lines.flatMap(_.split(" "))
val wordDstream = words.map(x => (x, 1))
val trackStateFunc = (batchTime: Time, word: String, one: Option[Int], state: State[Int]) => {
val sum = one.getOrElse(0) + state.getOption.getOrElse(0)
val output = (word, sum)
state.update(sum)
Some(output)
}
val stateDstream = wordDstream.trackStateByKey(
StateSpec.function(trackStateFunc).initialState(initialRDD))](https://image.slidesharecdn.com/sd2016-160512034804/75/Distributed-Real-Time-Stream-Processing-Why-and-How-2-0-62-2048.jpg)
![// Initial RDD input to updateStateByKey
val initialRDD = ssc.sparkContext.parallelize(List.empty[(String, Int)])
val lines = ...
val words = lines.flatMap(_.split(" "))
val wordDstream = words.map(x => (x, 1))
val trackStateFunc = (batchTime: Time, word: String, one: Option[Int], state: State[Int]) => {
val sum = one.getOrElse(0) + state.getOption.getOrElse(0)
val output = (word, sum)
state.update(sum)
Some(output)
}
val stateDstream = wordDstream.trackStateByKey(
StateSpec.function(trackStateFunc).initialState(initialRDD))
Spark Streaming](https://image.slidesharecdn.com/sd2016-160512034804/75/Distributed-Real-Time-Stream-Processing-Why-and-How-2-0-63-2048.jpg)
![// Initial RDD input to updateStateByKey
val initialRDD = ssc.sparkContext.parallelize(List.empty[(String, Int)])
val lines = ...
val words = lines.flatMap(_.split(" "))
val wordDstream = words.map(x => (x, 1))
val trackStateFunc = (batchTime: Time, word: String, one: Option[Int], state: State[Int]) => {
val sum = one.getOrElse(0) + state.getOption.getOrElse(0)
val output = (word, sum)
state.update(sum)
Some(output)
}
val stateDstream = wordDstream.trackStateByKey(
StateSpec.function(trackStateFunc).initialState(initialRDD))
Spark Streaming](https://image.slidesharecdn.com/sd2016-160512034804/75/Distributed-Real-Time-Stream-Processing-Why-and-How-2-0-64-2048.jpg)
![// Initial RDD input to updateStateByKey
val initialRDD = ssc.sparkContext.parallelize(List.empty[(String, Int)])
val lines = ...
val words = lines.flatMap(_.split(" "))
val wordDstream = words.map(x => (x, 1))
val trackStateFunc = (batchTime: Time, word: String, one: Option[Int], state: State[Int]) => {
val sum = one.getOrElse(0) + state.getOption.getOrElse(0)
val output = (word, sum)
state.update(sum)
Some(output)
}
val stateDstream = wordDstream.trackStateByKey(
StateSpec.function(trackStateFunc).initialState(initialRDD))
Spark Streaming](https://image.slidesharecdn.com/sd2016-160512034804/75/Distributed-Real-Time-Stream-Processing-Why-and-How-2-0-65-2048.jpg)
![Samza
class WordCountTask extends StreamTask with InitableTask {
private var store: CountStore = _
def init(config: Config, context: TaskContext) {
this.store = context.getStore("wordcount-store").asInstanceOf[KeyValueStore[String, Integer]]
}
override def process(envelope: IncomingMessageEnvelope,
collector: MessageCollector, coordinator: TaskCoordinator) {
val words = envelope.getMessage.asInstanceOf[String].split(" ")
words.foreach{ key =>
val count: Integer = Option(store.get(key)).getOrElse(0)
store.put(key, count + 1)
collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", "wordcount"), (key, count)))
}
}](https://image.slidesharecdn.com/sd2016-160512034804/75/Distributed-Real-Time-Stream-Processing-Why-and-How-2-0-66-2048.jpg)
![Samza
class WordCountTask extends StreamTask with InitableTask {
private var store: CountStore = _
def init(config: Config, context: TaskContext) {
this.store = context.getStore("wordcount-store").asInstanceOf[KeyValueStore[String, Integer]]
}
override def process(envelope: IncomingMessageEnvelope,
collector: MessageCollector, coordinator: TaskCoordinator) {
val words = envelope.getMessage.asInstanceOf[String].split(" ")
words.foreach{ key =>
val count: Integer = Option(store.get(key)).getOrElse(0)
store.put(key, count + 1)
collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", "wordcount"), (key, count)))
}
}](https://image.slidesharecdn.com/sd2016-160512034804/75/Distributed-Real-Time-Stream-Processing-Why-and-How-2-0-67-2048.jpg)
![Samza
class WordCountTask extends StreamTask with InitableTask {
private var store: CountStore = _
def init(config: Config, context: TaskContext) {
this.store = context.getStore("wordcount-store").asInstanceOf[KeyValueStore[String, Integer]]
}
override def process(envelope: IncomingMessageEnvelope,
collector: MessageCollector, coordinator: TaskCoordinator) {
val words = envelope.getMessage.asInstanceOf[String].split(" ")
words.foreach{ key =>
val count: Integer = Option(store.get(key)).getOrElse(0)
store.put(key, count + 1)
collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", "wordcount"), (key, count)))
}
}](https://image.slidesharecdn.com/sd2016-160512034804/75/Distributed-Real-Time-Stream-Processing-Why-and-How-2-0-68-2048.jpg)
![Apex
@ApplicationAnnotation(name="WordCount")
class Application extends StreamingApplication {
override def populateDAG(dag: DAG, configuration: Configuration): Unit = {
val input = dag.addOperator("input", new LineReader)
val parser = dag.addOperator("parser", new Parser)
val counter = dag.addOperator("counter", new UniqueCounter[String])
val out = dag.addOperator("console", new ConsoleOutputOperator)
dag.addStream[String]("lines", input.out, parser.in)
dag.addStream[String]("words", parser.out, counter.data)
dag.addStream[java.util.HashMap[String,Integer]]("counts", counter.count, out.input)
}
}](https://image.slidesharecdn.com/sd2016-160512034804/75/Distributed-Real-Time-Stream-Processing-Why-and-How-2-0-69-2048.jpg)
![Apex
@ApplicationAnnotation(name="WordCount")
class Application extends StreamingApplication {
override def populateDAG(dag: DAG, configuration: Configuration): Unit = {
val input = dag.addOperator("input", new LineReader)
val parser = dag.addOperator("parser", new Parser)
val counter = dag.addOperator("counter", new UniqueCounter[String])
val out = dag.addOperator("console", new ConsoleOutputOperator)
dag.addStream[String]("lines", input.out, parser.in)
dag.addStream[String]("words", parser.out, counter.data)
dag.addStream[java.util.HashMap[String,Integer]]("counts", counter.count, out.input)
}
}](https://image.slidesharecdn.com/sd2016-160512034804/75/Distributed-Real-Time-Stream-Processing-Why-and-How-2-0-70-2048.jpg)
![Flink
val env = ExecutionEnvironment.getExecutionEnvironment
val text = env.fromElements(...)
val words = text.flatMap ( _.split(" ") )
words.keyBy(x => x).mapWithState{
(word, count: Option[Int]) =>
{
val newCount = count.getOrElse(0) + 1
val output = (word, newCount)
(output, Some(newCount))
}
}
...](https://image.slidesharecdn.com/sd2016-160512034804/75/Distributed-Real-Time-Stream-Processing-Why-and-How-2-0-71-2048.jpg)
![Flink
val env = ExecutionEnvironment.getExecutionEnvironment
val text = env.fromElements(...)
val words = text.flatMap ( _.split(" ") )
words.keyBy(x => x).mapWithState{
(word, count: Option[Int]) =>
{
val newCount = count.getOrElse(0) + 1
val output = (word, newCount)
(output, Some(newCount))
}
}
...](https://image.slidesharecdn.com/sd2016-160512034804/75/Distributed-Real-Time-Stream-Processing-Why-and-How-2-0-72-2048.jpg)
![Project Maturity [Storm & Trident]
For a long time de-facto
industrial standard](https://image.slidesharecdn.com/sd2016-160512034804/75/Distributed-Real-Time-Stream-Processing-Why-and-How-2-0-79-2048.jpg)
![Project Maturity [Spark Streaming]
The most trending Scala repository these
days and one of the engines behind Scala’s
popularity](https://image.slidesharecdn.com/sd2016-160512034804/75/Distributed-Real-Time-Stream-Processing-Why-and-How-2-0-80-2048.jpg)
![Project Maturity [Samza]
Used by LinkedIn and also by tens of
other companies](https://image.slidesharecdn.com/sd2016-160512034804/75/Distributed-Real-Time-Stream-Processing-Why-and-How-2-0-81-2048.jpg)
![Project Maturity [Apex]
Graduated very recently, adopted by a
couple of corporate clients already](https://image.slidesharecdn.com/sd2016-160512034804/75/Distributed-Real-Time-Stream-Processing-Why-and-How-2-0-82-2048.jpg)
![Project Maturity [Flink]
Still an emerging project, but we can
see its first production deployments](https://image.slidesharecdn.com/sd2016-160512034804/75/Distributed-Real-Time-Stream-Processing-Why-and-How-2-0-83-2048.jpg)
![Recommendations [Storm & Trident]
● Fits for small and fast tasks
● Very low (tens of milliseconds) latency
● State & Fault tolerance degrades performance significantly
● Potential update to Heron
○ Keeps API, according to Twitter better in every single way
○ Future open-sourcing is uncertain](https://image.slidesharecdn.com/sd2016-160512034804/75/Distributed-Real-Time-Stream-Processing-Why-and-How-2-0-91-2048.jpg)
![Recommendations [Spark Streaming]
● Spark Ecosystem
● Data Exploration
● Latency is not critical
● Micro-batching limitations](https://image.slidesharecdn.com/sd2016-160512034804/75/Distributed-Real-Time-Stream-Processing-Why-and-How-2-0-92-2048.jpg)
![Recommendations [Samza]
● Kafka is a cornerstone of your architecture
● Application requires large states
● Don’t need exactly once
● Kafka Streams](https://image.slidesharecdn.com/sd2016-160512034804/75/Distributed-Real-Time-Stream-Processing-Why-and-How-2-0-93-2048.jpg)
![Recommendations [Apex]
● Prefer compositional approach
● Hadoop
● Great performance
● Dynamic DAG changes](https://image.slidesharecdn.com/sd2016-160512034804/75/Distributed-Real-Time-Stream-Processing-Why-and-How-2-0-94-2048.jpg)
![Recommendations [Flink]
● Conceptually great, fits very most use cases
● Take advantage of batch processing capabilities
● Need a functionality which is hard to implement in micro-batch
● Enough courage to use emerging project](https://image.slidesharecdn.com/sd2016-160512034804/75/Distributed-Real-Time-Stream-Processing-Why-and-How-2-0-95-2048.jpg)
Uploaded byPetr Zapletal
Distributed Real-Time Stream Processing: Why and How 2.0
The document discusses distributed real-time stream processing, detailing its importance due to the ever-increasing data generated, including frameworks, system comparisons, and use cases. It covers the pros and cons of various processing models and systems, including native streaming and micro-batching, highlighting their handling of state management, fault tolerance, and scalability. In addition, the document reviews popular stream processing frameworks such as Apache Storm, Spark Streaming, and Samza, along with their architecture and operational examples.
More Related Content
Similar to Distributed Real-Time Stream Processing: Why and How 2.0
![Governance, Deployment & Methodologies for Agentic Automation [2/3]](https://cdn.slidesharecdn.com/ss_thumbnails/agenticcommunityseries-day2-251112135038-a386476d-thumbnail.jpg?width=640&height=640&fit=bounds)
Distributed Real-Time Stream Processing: Why and How 2.0
- 1.
- 2. Petr Zapletal @petr_zapletal #ScalaDays @cakesolutions DistributedReal-Time Stream Processing: Why and How
- 3. Agenda ● Motivation ● StreamProcessing ● Available Frameworks ● Systems Comparison ● Recommendations
- 4. The Data Deluge ●New Sources and New Use Cases ● 8 Zettabytes (1 ZB = 1 trillion GB) created in 2015 ● Stream Processing to the Rescue
- 5. ● Continuous processing,aggregation and analysis of unbounded data Distributed Stream Processing
- 6. Points of Interest ➜Runtime and Programming Model
- 7. Points of Interest ➜Runtime and Programming Model ➜ Primitives
- 8. Points of Interest ➜Runtime and Programming Model ➜ Primitives ➜ State Management
- 9. Points of Interest ➜Runtime and Programming Model ➜ Primitives ➜ State Management ➜ Message Delivery Guarantees
- 10. Points of Interest ➜Runtime and Programming Model ➜ Primitives ➜ State Management ➜ Message Delivery Guarantees ➜ Fault Tolerance & Low Overhead Recovery
- 11. Points of Interest ➜Runtime and Programming Model ➜ Primitives ➜ State Management ➜ Message Delivery Guarantees ➜ Fault Tolerance & Low Overhead Recovery ➜ Latency, Throughput & Scalability
- 12. Points of Interest ➜Runtime and Programming Model ➜ Primitives ➜ State Management ➜ Message Delivery Guarantees ➜ Fault Tolerance & Low Overhead Recovery ➜ Latency, Throughput & Scalability ➜ Maturity and Adoption Level
- 13. Points of Interest ➜Runtime and Programming Model ➜ Primitives ➜ State Management ➜ Message Delivery Guarantees ➜ Fault Tolerance & Low Overhead Recovery ➜ Latency, Throughput & Scalability ➜ Maturity and Adoption Level ➜ Ease of Development and Operability
- 14. ● Most importanttrait of stream processing system ● Defines expressiveness, possible operations and its limitations ● Therefore defines systems capabilities and its use cases Runtime and Programming Model
- 15. Native Streaming records Sink Operator Source Operator Processing Operator records processedone at a time Processing Operator
- 16. Micro-batching records processed inshort batches Processing Operator Receiver records Processing Operator Micro-batches Sink Operator
- 17. Native Streaming ● Recordsare processed as they arrive Pros ⟹ Expressiveness ⟹ Low-latency ⟹ Stateful operations Cons ⟹ Throughput ⟹ Fault-tolerance is expensive ⟹ Load-balancing
- 18. Micro-batching Pros ⟹ High-throughput ⟹ Easierfault tolerance ⟹ Simpler load-balancing Cons ⟹ Lower latency, depends on batch interval ⟹ Limited expressivity ⟹ Harder stateful operations ● Splits incoming stream into small batches
- 19. Programming Model Compositional ⟹ Providesbasic building blocks as operators or sources ⟹ Custom component definition ⟹ Manual Topology definition & optimization ⟹ Advanced functionality often missing Declarative ⟹ High-level API ⟹ Operators as higher order functions ⟹ Abstract data types ⟹ Advance operations like state management or windowing supported out of the box ⟹ Advanced optimizers
- 20.
- 21. Storm Stor ● Pioneer inlarge scale stream processing
- 22. ● Higher levelmicro-batching system build atop Storm Trident
- 23. ● Unified batchand stream processing over a batch runtime Spark Streaming input data stream Spark Streaming Spark Engine batches of input data batches of processed data
- 24. Samza ● Builds heavilyon Kafka’s log based philosophy Task 1 Task 2 Task 3
- 25. Apex ● Processes massiveamounts of real-time events natively in Hadoop Operator Operator OperatorOperator Operator Operator Output Stream Stream Stream Stream Stream Tuple
- 26. Flink Stream Data Batch Data Kafka,RabbitMQ, ... HDFS, JDBC, ... ● Native streaming & High level API
- 27. System Comparison Native Micro-batchingNative Native Compositional Declarative Compositional Declarative At-least-once Exactly-once At-least-once Exactly-once Record ACKs RDD based Checkpointing Log-based Checkpointing Not build-in Dedicated Operators Stateful Operators Stateful Operators Very Low Medium Low Low Low Medium High High High High Medium Low Micro-batching Exactly-once* Dedicated DStream Medium High Streaming Model API Guarantees Fault Tolerance State Management Latency Throughput Maturity TRIDENT Hybrid Compositional* Exactly-once Checkpointing Stateful Operators Very Low High Medium
- 28. Counting Words ScalaDays 2016 ApacheApache Spark Storm Apache Trident Flink Streaming Samza Scala 2016 Streaming (Apache,3) (Streaming, 2) (2016, 2) (Spark, 1) (Storm, 1) (Trident, 1) (Flink, 1) (Samza, 1) (Scala, ) (ScalaDays, 1)
- 29. Storm TopologyBuilder builder =new TopologyBuilder(); builder.setSpout("spout", new RandomSentenceSpout(), 5); builder.setBolt("split", new Split(), 8).shuffleGrouping("spout"); builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word")); ... Map<String, Integer> counts = new HashMap<String, Integer>(); public void execute(Tuple tuple, BasicOutputCollector collector) { String word = tuple.getString(0); Integer count = counts.containsKey(word) ? counts.get(word) + 1 : 1; counts.put(word, count); collector.emit(new Values(word, count)); }
- 30. Storm TopologyBuilder builder =new TopologyBuilder(); builder.setSpout("spout", new RandomSentenceSpout(), 5); builder.setBolt("split", new Split(), 8).shuffleGrouping("spout"); builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word")); ... Map<String, Integer> counts = new HashMap<String, Integer>(); public void execute(Tuple tuple, BasicOutputCollector collector) { String word = tuple.getString(0); Integer count = counts.containsKey(word) ? counts.get(word) + 1 : 1; counts.put(word, count); collector.emit(new Values(word, count)); }
- 31. Storm TopologyBuilder builder =new TopologyBuilder(); builder.setSpout("spout", new RandomSentenceSpout(), 5); builder.setBolt("split", new Split(), 8).shuffleGrouping("spout"); builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word")); ... Map<String, Integer> counts = new HashMap<String, Integer>(); public void execute(Tuple tuple, BasicOutputCollector collector) { String word = tuple.getString(0); Integer count = counts.containsKey(word) ? counts.get(word) + 1 : 1; counts.put(word, count); collector.emit(new Values(word, count)); }
- 32. Trident public static StormTopologybuildTopology(LocalDRPC drpc) { FixedBatchSpout spout = ... TridentTopology topology = new TridentTopology(); TridentState wordCounts = topology.newStream("spout1", spout) .each(new Fields("sentence"),new Split(), new Fields("word")) .groupBy(new Fields("word")) .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("count")); ... }
- 33. Trident public static StormTopologybuildTopology(LocalDRPC drpc) { FixedBatchSpout spout = ... TridentTopology topology = new TridentTopology(); TridentState wordCounts = topology.newStream("spout1", spout) .each(new Fields("sentence"),new Split(), new Fields("word")) .groupBy(new Fields("word")) .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("count")); ... }
- 34. Trident public static StormTopologybuildTopology(LocalDRPC drpc) { FixedBatchSpout spout = ... TridentTopology topology = new TridentTopology(); TridentState wordCounts = topology.newStream("spout1", spout) .each(new Fields("sentence"),new Split(), new Fields("word")) .groupBy(new Fields("word")) .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("count")); ... }
- 35. Spark Streaming val conf= new SparkConf().setAppName("wordcount") val ssc = new StreamingContext(conf, Seconds(1)) val text = ... val counts = text.flatMap(line => line.split(" ")) .map(word => (word, 1)) .reduceByKey(_ + _) counts.print() ssc.start() ssc.awaitTermination()
- 36. Spark Streaming val conf= new SparkConf().setAppName("wordcount") val ssc = new StreamingContext(conf, Seconds(1)) val text = ... val counts = text.flatMap(line => line.split(" ")) .map(word => (word, 1)) .reduceByKey(_ + _) counts.print() ssc.start() ssc.awaitTermination()
- 37. val conf =new SparkConf().setAppName("wordcount") val ssc = new StreamingContext(conf, Seconds(1)) val text = ... val counts = text.flatMap(line => line.split(" ")) .map(word => (word, 1)) .reduceByKey(_ + _) counts.print() ssc.start() ssc.awaitTermination() Spark Streaming
- 38. val conf =new SparkConf().setAppName("wordcount") val ssc = new StreamingContext(conf, Seconds(1)) val text = ... val counts = text.flatMap(line => line.split(" ")) .map(word => (word, 1)) .reduceByKey(_ + _) counts.print() ssc.start() ssc.awaitTermination() Spark Streaming
- 39. Samza class WordCountTask extendsStreamTask { override def process(envelope: IncomingMessageEnvelope, collector: MessageCollector, coordinator: TaskCoordinator) { val text = envelope.getMessage.asInstanceOf[String] val counts = text.split(" ") .foldLeft(Map.empty[String, Int]) { (count, word) => count + (word -> (count.getOrElse(word, 0) + 1)) } collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", "wordcount"), counts)) }
- 40. Samza class WordCountTask extendsStreamTask { override def process(envelope: IncomingMessageEnvelope, collector: MessageCollector, coordinator: TaskCoordinator) { val text = envelope.getMessage.asInstanceOf[String] val counts = text.split(" ") .foldLeft(Map.empty[String, Int]) { (count, word) => count + (word -> (count.getOrElse(word, 0) + 1)) } collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", "wordcount"), counts)) }
- 41. class WordCountTask extendsStreamTask { override def process(envelope: IncomingMessageEnvelope, collector: MessageCollector, coordinator: TaskCoordinator) { val text = envelope.getMessage.asInstanceOf[String] val counts = text.split(" ") .foldLeft(Map.empty[String, Int]) { (count, word) => count + (word -> (count.getOrElse(word, 0) + 1)) } collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", "wordcount"), counts)) } Samza
- 42. Apex val input =dag.addOperator("input", new LineReader) val parser = dag.addOperator("parser", new Parser) val out = dag.addOperator("console", new ConsoleOutputOperator) dag.addStream[String]("lines", input.out, parser.in) dag.addStream[String]("words", parser.out, counter.data) class Parser extends BaseOperator { @transient val out = new DefaultOutputPort[String]() @transient val in = new DefaultInputPort[String]() override def process(t: String): Unit = { for(w <- t.split(" ")) out.emit(w) } }
- 43. Apex val input =dag.addOperator("input", new LineReader) val parser = dag.addOperator("parser", new Parser) val out = dag.addOperator("console", new ConsoleOutputOperator) dag.addStream[String]("lines", input.out, parser.in) dag.addStream[String]("words", parser.out, counter.data) class Parser extends BaseOperator { @transient val out = new DefaultOutputPort[String]() @transient val in = new DefaultInputPort[String]() override def process(t: String): Unit = { for(w <- t.split(" ")) out.emit(w) } }
- 44. val input =dag.addOperator("input", new LineReader) val parser = dag.addOperator("parser", new Parser) val out = dag.addOperator("console", new ConsoleOutputOperator) dag.addStream[String]("lines", input.out, parser.in) dag.addStream[String]("words", parser.out, counter.data) class Parser extends BaseOperator { @transient val out = new DefaultOutputPort[String]() @transient val in = new DefaultInputPort[String]() override def process(t: String): Unit = { for(w <- t.split(" ")) out.emit(w) } } Apex
- 45. Flink val env =ExecutionEnvironment.getExecutionEnvironment val text = env.fromElements(...) val counts = text.flatMap ( _.split(" ") ) .map ( (_, 1) ) .groupBy(0) .sum(1) counts.print() env.execute("wordcount")
- 46. Flink val env =ExecutionEnvironment.getExecutionEnvironment val text = env.fromElements(...) val counts = text.flatMap ( _.split(" ") ) .map ( (_, 1) ) .groupBy(0) .sum(1) counts.print() env.execute("wordcount")
- 47. Fault tolerance instreaming systems is inherently harder that in batch Fault Tolerance
- 48. Storm & Trident Acksare delivered via a system-level bolt ack ack Acker Bolt {A} {B} ACK
- 49. Spark Streaming faster recoveryby using multiple nodes for recomputations failed tasks failed node
- 50. Samza inputstream Checkpoint partition 0: offset6 partition 1: offset 4 partition 2: offset 8 Partition 0 Partition 1 Partition 2 Samza StreamTask partition 1 StreamTask partition 2 StreamTask partition 0
- 51. Apex Aggregated Application Windows SystemEvent Window t=0.5 sec. t=0.5 sec. t=0.5 sec. Sliding Application Windows BeginWindow Checkpoint EndWindow BeginWindow EndWindow Checkpoint BeginWindow EndWindow BeginWindow EndWindow
- 52. Flink data stream checkpoint barrier n checkpoint barriern-1 part of checkpoint n+1 part of checkpoint n part of checkpoint n-1 newer records older records
- 53. Managing State f: (input,state) => (output, state’)
- 54.
- 55. Spark Streaming Input Stream JobJob Job Output Stream State Micro-batch processing
- 56.
- 57.
- 58.
- 59. Counting Words Revisited ScalaDays2016 Apache Apache Spark Storm Apache Trident Flink Streaming Samza Scala 2016 Streaming (Apache,3) (Streaming, 2) (2016, 2) (Spark, 1) (Storm, 1) (Trident, 1) (Flink, 1) (Samza, 1) (Scala, ) (ScalaDays, 1)
- 60. Trident import storm.trident.operation.builtin.Count; TridentTopology topology= new TridentTopology(); TridentState wordCounts = topology.newStream("spout1", spout) .each(new Fields("sentence"), new Split(), new Fields("word")) .groupBy(new Fields("word")) .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("count")) .parallelismHint(6);
- 61. Trident import storm.trident.operation.builtin.Count; TridentTopology topology= new TridentTopology(); TridentState wordCounts = topology.newStream("spout1", spout) .each(new Fields("sentence"), new Split(), new Fields("word")) .groupBy(new Fields("word")) .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("count")) .parallelismHint(6);
- 62. Spark Streaming // InitialRDD input to updateStateByKey val initialRDD = ssc.sparkContext.parallelize(List.empty[(String, Int)]) val lines = ... val words = lines.flatMap(_.split(" ")) val wordDstream = words.map(x => (x, 1)) val trackStateFunc = (batchTime: Time, word: String, one: Option[Int], state: State[Int]) => { val sum = one.getOrElse(0) + state.getOption.getOrElse(0) val output = (word, sum) state.update(sum) Some(output) } val stateDstream = wordDstream.trackStateByKey( StateSpec.function(trackStateFunc).initialState(initialRDD))
- 63. // Initial RDDinput to updateStateByKey val initialRDD = ssc.sparkContext.parallelize(List.empty[(String, Int)]) val lines = ... val words = lines.flatMap(_.split(" ")) val wordDstream = words.map(x => (x, 1)) val trackStateFunc = (batchTime: Time, word: String, one: Option[Int], state: State[Int]) => { val sum = one.getOrElse(0) + state.getOption.getOrElse(0) val output = (word, sum) state.update(sum) Some(output) } val stateDstream = wordDstream.trackStateByKey( StateSpec.function(trackStateFunc).initialState(initialRDD)) Spark Streaming
- 64. // Initial RDDinput to updateStateByKey val initialRDD = ssc.sparkContext.parallelize(List.empty[(String, Int)]) val lines = ... val words = lines.flatMap(_.split(" ")) val wordDstream = words.map(x => (x, 1)) val trackStateFunc = (batchTime: Time, word: String, one: Option[Int], state: State[Int]) => { val sum = one.getOrElse(0) + state.getOption.getOrElse(0) val output = (word, sum) state.update(sum) Some(output) } val stateDstream = wordDstream.trackStateByKey( StateSpec.function(trackStateFunc).initialState(initialRDD)) Spark Streaming
- 65. // Initial RDDinput to updateStateByKey val initialRDD = ssc.sparkContext.parallelize(List.empty[(String, Int)]) val lines = ... val words = lines.flatMap(_.split(" ")) val wordDstream = words.map(x => (x, 1)) val trackStateFunc = (batchTime: Time, word: String, one: Option[Int], state: State[Int]) => { val sum = one.getOrElse(0) + state.getOption.getOrElse(0) val output = (word, sum) state.update(sum) Some(output) } val stateDstream = wordDstream.trackStateByKey( StateSpec.function(trackStateFunc).initialState(initialRDD)) Spark Streaming
- 66. Samza class WordCountTask extendsStreamTask with InitableTask { private var store: CountStore = _ def init(config: Config, context: TaskContext) { this.store = context.getStore("wordcount-store").asInstanceOf[KeyValueStore[String, Integer]] } override def process(envelope: IncomingMessageEnvelope, collector: MessageCollector, coordinator: TaskCoordinator) { val words = envelope.getMessage.asInstanceOf[String].split(" ") words.foreach{ key => val count: Integer = Option(store.get(key)).getOrElse(0) store.put(key, count + 1) collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", "wordcount"), (key, count))) } }
- 67. Samza class WordCountTask extendsStreamTask with InitableTask { private var store: CountStore = _ def init(config: Config, context: TaskContext) { this.store = context.getStore("wordcount-store").asInstanceOf[KeyValueStore[String, Integer]] } override def process(envelope: IncomingMessageEnvelope, collector: MessageCollector, coordinator: TaskCoordinator) { val words = envelope.getMessage.asInstanceOf[String].split(" ") words.foreach{ key => val count: Integer = Option(store.get(key)).getOrElse(0) store.put(key, count + 1) collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", "wordcount"), (key, count))) } }
- 68. Samza class WordCountTask extendsStreamTask with InitableTask { private var store: CountStore = _ def init(config: Config, context: TaskContext) { this.store = context.getStore("wordcount-store").asInstanceOf[KeyValueStore[String, Integer]] } override def process(envelope: IncomingMessageEnvelope, collector: MessageCollector, coordinator: TaskCoordinator) { val words = envelope.getMessage.asInstanceOf[String].split(" ") words.foreach{ key => val count: Integer = Option(store.get(key)).getOrElse(0) store.put(key, count + 1) collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", "wordcount"), (key, count))) } }
- 69. Apex @ApplicationAnnotation(name="WordCount") class Application extendsStreamingApplication { override def populateDAG(dag: DAG, configuration: Configuration): Unit = { val input = dag.addOperator("input", new LineReader) val parser = dag.addOperator("parser", new Parser) val counter = dag.addOperator("counter", new UniqueCounter[String]) val out = dag.addOperator("console", new ConsoleOutputOperator) dag.addStream[String]("lines", input.out, parser.in) dag.addStream[String]("words", parser.out, counter.data) dag.addStream[java.util.HashMap[String,Integer]]("counts", counter.count, out.input) } }
- 70. Apex @ApplicationAnnotation(name="WordCount") class Application extendsStreamingApplication { override def populateDAG(dag: DAG, configuration: Configuration): Unit = { val input = dag.addOperator("input", new LineReader) val parser = dag.addOperator("parser", new Parser) val counter = dag.addOperator("counter", new UniqueCounter[String]) val out = dag.addOperator("console", new ConsoleOutputOperator) dag.addStream[String]("lines", input.out, parser.in) dag.addStream[String]("words", parser.out, counter.data) dag.addStream[java.util.HashMap[String,Integer]]("counts", counter.count, out.input) } }
- 71. Flink val env =ExecutionEnvironment.getExecutionEnvironment val text = env.fromElements(...) val words = text.flatMap ( _.split(" ") ) words.keyBy(x => x).mapWithState{ (word, count: Option[Int]) => { val newCount = count.getOrElse(0) + 1 val output = (word, newCount) (output, Some(newCount)) } } ...
- 72. Flink val env =ExecutionEnvironment.getExecutionEnvironment val text = env.fromElements(...) val words = text.flatMap ( _.split(" ") ) words.keyBy(x => x).mapWithState{ (word, count: Option[Int]) => { val newCount = count.getOrElse(0) + 1 val output = (word, newCount) (output, Some(newCount)) } } ...
- 73. Performance Hard to designnot biased test, lots of variables
- 74. Performance ➜ Latency vs.Throughput
- 75. Performance ➜ Latency vs.Throughput ➜ Costs of Delivery Guarantees, Fault-tolerance & State Management
- 76. Performance ➜ Latency vs.Throughput ➜ Costs of Delivery Guarantees, Fault-tolerance & State Management ➜ Tuning
- 77. Performance ➜ Latency vs.Throughput ➜ Costs of Delivery Guarantees, Fault-tolerance & State Management ➜ Tuning ➜ Network operations, Data locality & Serialization
- 78. Project Maturity When pickingup the framework, you should always consider its maturity
- 79. Project Maturity [Storm& Trident] For a long time de-facto industrial standard
- 80. Project Maturity [SparkStreaming] The most trending Scala repository these days and one of the engines behind Scala’s popularity
- 81. Project Maturity [Samza] Usedby LinkedIn and also by tens of other companies
- 82. Project Maturity [Apex] Graduatedvery recently, adopted by a couple of corporate clients already
- 83. Project Maturity [Flink] Stillan emerging project, but we can see its first production deployments
- 84. Summary Native Micro-batching NativeNative Compositional Declarative Compositional Declarative At-least-once Exactly-once At-least-once Exactly-once Record ACKs RDD based Checkpointing Log-based Checkpointing Not build-in Dedicated Operators Stateful Operators Stateful Operators Very Low Medium Low Low Low Medium High High High High Medium Low Micro-batching Exactly-once* Dedicated DStream Medium High Streaming Model API Guarantees Fault Tolerance State Management Latency Throughput Maturity TRIDENT Hybrid Compositional* Exactly-once Checkpointing Stateful Operators Very Low High Medium
- 85. General Guidelines As alwaysit depends
- 86. General Guidelines ➜ Evaluateparticular application needs
- 87. General Guidelines ➜ Evaluateparticular application needs ➜ Programming model
- 88. General Guidelines ➜ Evaluateparticular application needs ➜ Programming model ➜ Available delivery guarantees
- 89. General Guidelines ➜ Evaluateparticular application needs ➜ Programming model ➜ Available delivery guarantees ➜ Almost all non-trivial jobs have state
- 90. General Guidelines ➜ Evaluateparticular application needs ➜ Programming model ➜ Available delivery guarantees ➜ Almost all non-trivial jobs have state ➜ Fast recovery is critical
- 91. Recommendations [Storm &Trident] ● Fits for small and fast tasks ● Very low (tens of milliseconds) latency ● State & Fault tolerance degrades performance significantly ● Potential update to Heron ○ Keeps API, according to Twitter better in every single way ○ Future open-sourcing is uncertain
- 92. Recommendations [Spark Streaming] ●Spark Ecosystem ● Data Exploration ● Latency is not critical ● Micro-batching limitations
- 93. Recommendations [Samza] ● Kafkais a cornerstone of your architecture ● Application requires large states ● Don’t need exactly once ● Kafka Streams
- 94. Recommendations [Apex] ● Prefercompositional approach ● Hadoop ● Great performance ● Dynamic DAG changes
- 95. Recommendations [Flink] ● Conceptuallygreat, fits very most use cases ● Take advantage of batch processing capabilities ● Need a functionality which is hard to implement in micro-batch ● Enough courage to use emerging project
- 96. Dataflow and ApacheBeam Dataflow Model & SDKs Apache Flink Apache Spark Direct Pipeline Google Cloud Dataflow Stream Processing Batch Processing Multiple Modes One Pipeline Many Runtimes Local or cloud Local Cloud
- 97.
- 98. MANCHESTER LONDON NEWYORK @petr_zapletal @cakesolutions 347 708 1518 [email protected] We are hiring http://www.cakesolutions.net/careers