Package com.linkedin.davinci.kafka.consumer

Class PartitionConsumptionState

java.lang.Object
com.linkedin.davinci.kafka.consumer.PartitionConsumptionState
public class PartitionConsumptionState extends Object
This class is used to maintain internal state for consumption of each partition.

  • Constructor Details

    • PartitionConsumptionState

      public PartitionConsumptionState(String replicaId, int partition, OffsetRecord offsetRecord, boolean hybrid)

  • Method Details

    • getPartition

      public int getPartition()

    • getLastVTProduceCallFuture

      public CompletableFuture<Void> getLastVTProduceCallFuture()

    • setLastVTProduceCallFuture

      public void setLastVTProduceCallFuture(CompletableFuture<Void> lastVTProduceCallFuture)

    • getOffsetRecord

      public OffsetRecord getOffsetRecord()

    • setDeferredWrite

      public void setDeferredWrite(boolean deferredWrite)

    • isDeferredWrite

      public boolean isDeferredWrite()

    • isStarted

      public boolean isStarted()

    • isEndOfPushReceived

      public final boolean isEndOfPushReceived()

    • isWaitingForReplicationLag

      public boolean isWaitingForReplicationLag()

    • lagHasCaughtUp

      public void lagHasCaughtUp()

    • hasLagCaughtUp

      public boolean hasLagCaughtUp()

    • isNearlineMetricsRecordingValid

      public boolean isNearlineMetricsRecordingValid(long producerTimeStampInMs)
      check to ignore calculating latency from pubsub broker to ready to serve for messages that are getting caught up from previous pushes.
      Parameters:
      producerTimeStampInMs - timestamp of the message

    • isCompletionReported

      public boolean isCompletionReported()

    • completionReported

      public void completionReported()

    • isSubscribed

      public boolean isSubscribed()

    • unsubscribe

      public void unsubscribe()

    • isLatchReleased

      public boolean isLatchReleased()

    • releaseLatch

      public void releaseLatch()

    • errorReported

      public void errorReported()

    • isErrorReported

      public boolean isErrorReported()

    • isComplete

      public boolean isComplete()

    • isHybrid

      public final boolean isHybrid()

    • isBatchOnly

      public boolean isBatchOnly()

    • toString

      public String toString()
      Overrides:
      toString in class Object

    • getProcessedRecordSizeSinceLastSync

      public long getProcessedRecordSizeSinceLastSync()

    • incrementProcessedRecordSizeSinceLastSync

      public void incrementProcessedRecordSizeSinceLastSync(int recordSize)

    • resetProcessedRecordSizeSinceLastSync

      public void resetProcessedRecordSizeSinceLastSync()

    • setLeaderFollowerState

      public void setLeaderFollowerState(LeaderFollowerStateType state)

    • getLeaderFollowerState

      public final LeaderFollowerStateType getLeaderFollowerState()

    • setLastLeaderPersistFuture

      public void setLastLeaderPersistFuture(Future<Void> future)

    • getLastLeaderPersistFuture

      public Future<Void> getLastLeaderPersistFuture()

    • getLastQueuedRecordPersistedFuture

      public CompletableFuture<Void> getLastQueuedRecordPersistedFuture()

    • setLastQueuedRecordPersistedFuture

      public void setLastQueuedRecordPersistedFuture(CompletableFuture<Void> lastQueuedRecordPersistedFuture)

    • setTopicSwitch

      public void setTopicSwitch(TopicSwitchWrapper topicSwitch)
      Update the in-memory state for TopicSwitch whenever encounter a new TopicSwitch message or after a restart.

    • getTopicSwitch

      public TopicSwitchWrapper getTopicSwitch()

    • setConsumeRemotely

      public void setConsumeRemotely(boolean isConsumingRemotely)

    • consumeRemotely

      public boolean consumeRemotely()

    • initializeExpectedChecksum

      public void initializeExpectedChecksum()

    • finalizeExpectedChecksum

      public void finalizeExpectedChecksum()

    • getVeniceWriterLazyRef

      public Lazy<VeniceWriter<byte[],byte[],byte[]>> getVeniceWriterLazyRef()

    • setVeniceWriterLazyRef

      public void setVeniceWriterLazyRef(Lazy<VeniceWriter<byte[],byte[],byte[]>> veniceWriterLazyRef)

    • maybeUpdateExpectedChecksum

      public void maybeUpdateExpectedChecksum(byte[] key, Put put)
      Keep updating the checksum for key/value pair received from kafka PUT message. If the checksum instance is not configured via initializeExpectedChecksum() then do nothing. This api will keep the caller's code clean.
      Parameters:
      key -
      put -

    • resetExpectedChecksum

      public void resetExpectedChecksum()

    • getExpectedChecksum

      public byte[] getExpectedChecksum()

    • getLatestMessageConsumedTimestampInMs

      public long getLatestMessageConsumedTimestampInMs()

    • setLatestMessageConsumedTimestampInMs

      public void setLatestMessageConsumedTimestampInMs(long consumedTimestampInMs)

    • getLatestPolledMessageTimestampInMs

      public long getLatestPolledMessageTimestampInMs()

    • setLatestPolledMessageTimestampInMs

      public void setLatestPolledMessageTimestampInMs(long timestampInMs)

    • getConsumptionStartTimeInMs

      public long getConsumptionStartTimeInMs()

    • setTransientRecord

      public void setTransientRecord(int kafkaClusterId, long kafkaConsumedOffset, byte[] key, int valueSchemaId, org.apache.avro.generic.GenericRecord replicationMetadataRecord)

    • setTransientRecord

      public void setTransientRecord(int kafkaClusterId, long kafkaConsumedOffset, byte[] key, byte[] value, int valueOffset, int valueLen, int valueSchemaId, org.apache.avro.generic.GenericRecord replicationMetadataRecord)

    • getTransientRecord

      public PartitionConsumptionState.TransientRecord getTransientRecord(byte[] key)

    • mayRemoveTransientRecord

      public PartitionConsumptionState.TransientRecord mayRemoveTransientRecord(int kafkaClusterId, long kafkaConsumedOffset, byte[] key)
      This operation is performed atomically to delete the record only when the provided sourceOffset matches.
      Parameters:
      kafkaClusterId -
      kafkaConsumedOffset -
      key -
      Returns:

    • getSourceTopicPartition

      public PubSubTopicPartition getSourceTopicPartition(PubSubTopic topic)

    • getTransientRecordMapSize

      public int getTransientRecordMapSize()

    • skipKafkaMessage

      public boolean skipKafkaMessage()

    • setSkipKafkaMessage

      public void setSkipKafkaMessage(boolean skipKafkaMessage)

    • recordReadyToServeInOffsetRecord

      public void recordReadyToServeInOffsetRecord()
      This persists to the offsetRecord associated to this partitionConsumptionState that the ready to serve check has passed. This will be persisted to disk once the offsetRecord is checkpointed, and subsequent restarts will consult this information when determining if the node should come online or not to serve traffic

    • getReadyToServeInOffsetRecord

      public boolean getReadyToServeInOffsetRecord()

    • updateLeaderConsumedUpstreamRTOffset

      public void updateLeaderConsumedUpstreamRTOffset(String kafkaUrl, long offset)

    • getLeaderConsumedUpstreamRTOffset

      public long getLeaderConsumedUpstreamRTOffset(String kafkaUrl)

    • updateLatestProcessedUpstreamRTOffset

      public void updateLatestProcessedUpstreamRTOffset(String kafkaUrl, long offset)

    • updateLatestRTOffsetTriedToProduceToVTMap

      public void updateLatestRTOffsetTriedToProduceToVTMap(String kafkaUrl, long offset)

    • getLatestRTOffsetTriedToProduceToVTMap

      public long getLatestRTOffsetTriedToProduceToVTMap(String kafkaUrl)

    • updateLatestIgnoredUpstreamRTOffset

      public void updateLatestIgnoredUpstreamRTOffset(String kafkaUrl, long offset)

    • getLatestIgnoredUpstreamRTOffset

      public long getLatestIgnoredUpstreamRTOffset(String kafkaUrl)

    • getLatestProcessedUpstreamRTOffsetWithIgnoredMessages

      public long getLatestProcessedUpstreamRTOffsetWithIgnoredMessages(String kafkaUrl)

    • getLatestProcessedUpstreamRTOffset

      public long getLatestProcessedUpstreamRTOffset(String kafkaUrl)

    • getLatestProcessedUpstreamRTOffsetWithNoDefault

      public Long getLatestProcessedUpstreamRTOffsetWithNoDefault(String kafkaUrl)

    • getLeaderOffset

      public long getLeaderOffset(String kafkaURL, PubSubTopicRepository pubSubTopicRepository)
      The caller of this API should be interested in which offset currently leader should consume from now. 1. If currently leader should consume from real-time topic, return upstream RT offset; 2. if currently leader should consume from version topic, return either remote VT offset or local VT offset, depending on whether the remote consumption flag is on.

    • setStartOfPushTimestamp

      public void setStartOfPushTimestamp(long startOfPushTimestamp)

    • getStartOfPushTimestamp

      public long getStartOfPushTimestamp()

    • setEndOfPushTimestamp

      public void setEndOfPushTimestamp(long endOfPushTimestamp)

    • getEndOfPushTimestamp

      public long getEndOfPushTimestamp()

    • updateLatestProcessedLocalVersionTopicOffset

      public void updateLatestProcessedLocalVersionTopicOffset(long offset)

    • getLatestProcessedLocalVersionTopicOffset

      public long getLatestProcessedLocalVersionTopicOffset()

    • updateLatestProcessedUpstreamVersionTopicOffset

      public void updateLatestProcessedUpstreamVersionTopicOffset(long offset)

    • getLatestProcessedUpstreamVersionTopicOffset

      public long getLatestProcessedUpstreamVersionTopicOffset()

    • setDataRecoveryCompleted

      public void setDataRecoveryCompleted(boolean dataRecoveryCompleted)

    • isDataRecoveryCompleted

      public boolean isDataRecoveryCompleted()

    • getLatestProcessedUpstreamRTOffsetMap

      public Map<String,Long> getLatestProcessedUpstreamRTOffsetMap()

    • getLeaderGUID

      public GUID getLeaderGUID()

    • setLeaderGUID

      public void setLeaderGUID(GUID leaderGUID)

    • getLeaderHostId

      public String getLeaderHostId()

    • setLeaderHostId

      public void setLeaderHostId(String hostId)

    • isLeaderCompleted

      public boolean isLeaderCompleted()

    • getLeaderCompleteState

      public LeaderCompleteState getLeaderCompleteState()

    • setLeaderCompleteState

      public void setLeaderCompleteState(LeaderCompleteState leaderCompleteState)

    • getLastLeaderCompleteStateUpdateInMs

      public long getLastLeaderCompleteStateUpdateInMs()

    • setLastLeaderCompleteStateUpdateInMs

      public void setLastLeaderCompleteStateUpdateInMs(long lastLeaderCompleteStateUpdateInMs)

    • getReplicaId

      public String getReplicaId()

    • addIncPushVersionToPendingReportList

      public void addIncPushVersionToPendingReportList(String incPushVersion)

    • getPendingReportIncPushVersionList

      public List<String> getPendingReportIncPushVersionList()

    • clearPendingReportIncPushVersionList

      public void clearPendingReportIncPushVersionList()