Data.  Data streamed in topics

Topics split into partitions

Brokers.  Replicated broker cluster

Leader follower architecture

Clients.  Producers

Consumers

Stream processors

public class KafkaProducer <K,V> implements Producer <K,V> {
    public KafkaProducer (Properties properties) { ... }

    public Future <RecordMetadata> send (ProducerRecord <K,V> record) { ... }
    public Future <RecordMetadata> send (ProducerRecord <K,V> record, Callback callback) { ... }

    public void flush () { ... }
    public void close () { ... }

    public void initTransactions () { ... }
    public void beginTransaction () { ... }
    public void abortTransaction () { ... }
    public void commitTransaction () { ... }

    // Introspection.
    public List <PartitionInfo> partitionsFor (String topic) { ... }

    ...
}

public class ProducerRecord <K,V> {
    public ProducerRecord (String topic, V value) { ... }
    public ProducerRecord (String topic, K key, V value) { ... }
    public ProducerRecord (
        String topic, Integer partition, K key, V value) { ... }
    public ProducerRecord (
        String topic, Integer partition, K key, V value, Iterable <Header> headers) { ... }
    public ProducerRecord (
        String topic, Integer partition, Long timestamp, K key, V value, Iterable <Header> headers) { ... }

    public K key () { ... }
    public V value () { ... }
    public Headers headers () { ... }

    ...
}

public interface Header {
    String key ();
    byte [] value ();
}

public final class RecordMetadata {
    public boolean hasOffset () { ... }
    public long offset () { ... }
    public boolean hasTimestamp () { ... }
    public long timestamp () { ... }

    public String topic () { ... }
    public int partition () { ... }

    public int serializedKeySize () { ... }
    public int serializedValueSize () { ... }
}

public interface Callback {
    void onCompletion (RecordMetadata metadata, Exception exception);
}

public class KafkaConsumer <K,V> implements Consumer <K,V> {
    public KafkaConsumer (Properties properties) { ... }

    // Statically assigned topics and partitions.
    public void assign (Collection <TopicPartition> partitions) { ... }
    public Set <TopicPartition> assignment () { ... }

    // Specific topics with dynamically assigned partitions.
    public void subscribe (Collection <String> topics) { ... }
    public void subscribe (Collection <String> topics, ConsumerRebalanceListener listener) { ... }

    // Regular expression topics with dynamically assigned partitions.
    public void subscribe (Pattern pattern) { ... }
    public void subscribe (Pattern pattern, ConsumerRebalanceListener listener) { ... }

    public void unsubscribe() { ... }
    public Set <String> subscription () { ... }

    // Poll for records.
    public ConsumerRecords <K,V> poll (final Duration timeout) { ... }

    // Seek and query position in topic partitions.
    public void seek (TopicPartition partition, long offset) { ... }
    public void seek (TopicPartition partition, OffsetAndMetadata offsetAndMetadata) { ... }
    public void seekToEnd (Collection <TopicPartition> partitions) { ... }
    public void seekToBeginning (Collection <TopicPartition> partitions) { ... }

    public long position (TopicPartition partition) { ... }
    public long position (TopicPartition partition, final Duration timeout) { ... }

    // Set and query committed position in topic partitions.
    public void commitSync () { ... }
    public void commitSync (Duration timeout) { ... }
    public void commitSync (final Map <TopicPartition, OffsetAndMetadata> offsets) { ... }
    public void commitSync (final Map <TopicPartition, OffsetAndMetadata> offsets, final Duration timeout) { ... }
    public void commitAsync () { ... }
    public void commitAsync (OffsetCommitCallback callback) { ... }

    public OffsetAndMetadata committed (TopicPartition partition) { ... }
    public OffsetAndMetadata committed (TopicPartition partition, final Duration timeout) { ... }

    public void pause (Collection<TopicPartition> partitions) { ... }
    public void resume (Collection<TopicPartition> partitions) { ... }
    public void close () { ... }

    // Introspection.
    public Map <String, List <PartitionInfo>> listTopics () { ... }
    public List <PartitionInfo> partitionsFor (String topic) { ... }

    ...
}

public class ConsumerRecord <K,V> {
    public String topic () { ... }
    public int partition () { ... }
    public long offset () { ... }
    public long timestamp () { ... }

    public K key () { ... }
    public V value () { ... }
    public Headers headers () { ... }

    public int serializedKeySize () { ... }
    public int serializedValueSize () { ... }

    ...
}

public interface KStream <K,V> {

    // Filter stream by predicate.
    KStream <K,V> filter (Predicate <? super K, ? super V> predicate);
    KStream <K,V> filterNot (Predicate <? super K, ? super V> predicate);

    // Replace key with new key.
    <KR> KStream <KR,V> selectKey (KeyValueMapper <? super K, ? super V, ? extends KR> mapper);

    // Map entry to new entry.
    <KR,VR> KStream <KR,VR> map (KeyValueMapper <
        ? super K, ? super V,
        ? extends KeyValue <? extends KR, ? extends VR>> mapper);

    // Map value to new value.
    <VR> KStream <K,VR> mapValues (ValueMapper <? super V, ? extends VR> mapper);

    // Map entry to multiple new entries.
    <KR,VR> KStream <KR,VR> flatMap (KeyValueMapper <
        ? super K, ? super V,
        ? extends Iterable <? extends KeyValue <? extends KR, ? extends VR>> mapper);

    // Map value to multiple new values.
    <VR> KStream <K,VR> flatMapValues (ValueMapper <? super V, ? extends Iterable <? extends VR>> mapper);

    // Print entries.
    void print (Printed <K, V> printed);

    // Consume or peek at entries with action.
    void foreach (ForeachAction <? super K, ? super V> action);
    KStream <K,V> peek (ForeachAction <? super K, ? super V> action);

    // Split by predicate or merge a stream.
    KStream <K,V> [] branch (Predicate <? super K, ? super V> ... predicates);
    KStream <K,V> merge (KStream <K,V> stream);

    // Materialize a stream into a topic.
    KStream <K,V> through (String topic);
    void to (String topic);

    // Process a stream using a stateful processor.
    <KO,VO> KStream <KO,VO> process (
        ProcessorSupplier <? super K, ? super V, KO, VO> processorSupplier,
        String... stateStoreNames);
    <VO> KStream <K,VO> processValues (
        FixedKeyProcessorSupplier <? super K, ? super V, VO> processorSupplier,
        String... stateStoreNames);

    // Group entries in a stream.
    KGroupedStream <K,V> groupByKey ();
    <KR> KGroupedStream <KR,V> groupBy (KeyValueMapper <? super K, ? super V, KR> selector);

    // Join stream with another stream or table on key.
    // Operation on streams limited by join window.
    <VO,VR> KStream <K,VR> join (
        KStream <K, VO> otherStream,
        ValueJoiner <? super V, ? super VO, ? extends VR> joiner,
        JoinWindows windows);
    <VO,VR> KStream <K,VR> leftJoin (
        KStream <K, VO> otherStream,
        ValueJoiner <? super V, ? super VO, ? extends VR> joiner,
        JoinWindows windows);
    <VO,VR> KStream <K,VR> outerJoin (
        KStream <K,VO> otherStream,
        ValueJoiner <? super V, ? super VO, ? extends VR> joiner,
        JoinWindows windows);
    <VT,VR> KStream <K,VR> join (
        KTable <K,VT> table,
        ValueJoiner <? super V, ? super VT, ? extends VR> joiner,
        Joined <K, V, VT> joined);
    <VT,VR> KStream <K,VR> leftJoin (
        KTable <K,VT> table,
        ValueJoiner <? super V, ? super VT, ? extends VR> joiner);

    ...
}

public interface KGroupedStream <K,V> {

    KTable <K,Long> count ();
    KTable <K,V> reduce (Reducer <V> reducer);

    <VR> KTable <K,VR> aggregate (
        Initializer <VR> initializer,
        Aggregator <? super K, ? super V, VR> aggregator);

    <W extends Window> TimeWindowedKStream <K,V> windowedBy (Windows<W> windows);

    ...
}

public interface Processor <KIn, VIn, KOut, VOut> {

    // Lifecycle.
    default void init (final ProcessorContext <KOut, VOut> context) {}
    default void close () {}

    // Process individual records.
    void process (Record<KIn, VIn> record);
}


public interface ProcessorContext <KForward, VForward> extends ProcessingContext {

    // Forward record to all child processors.
    <K extends KForward, V extends VForward> void forward (Record <K, V> record);

    // Forward record to specified child processor.
    <K extends KForward, V extends VForward> void forward (Record <K, V> record, final String childName);
}

public interface ReadOnlyKeyValueStore <K, V> {

    V get (K key);

    KeyValueIterator <K, V> range (K from, K to);
    default KeyValueIterator <K, V> reverseRange (K from, K to) { ... }

    KeyValueIterator <K, V> all ();
    default KeyValueIterator <K, V> reverseAll () { ... }

    default <PS extends Serializer <P>, P> KeyValueIterator <K, V> prefixScan (P prefix, PS prefixKeySerializer) { ... }

    long approximateNumEntries ();
}


public interface KeyValueStore <K, V> extends StateStore, ReadOnlyKeyValueStore <K, V> {

    void put (K key, V value);
    V putIfAbsent (K key, V value);
    void putAll (List <KeyValue <K, V>> entries);

    V delete (K key);
}