Class PartitionConsumptionState


  • public class PartitionConsumptionState
    extends java.lang.Object
    This class is used to maintain internal state for consumption of each partition.
    • Constructor Detail

      • PartitionConsumptionState

        public PartitionConsumptionState​(int partition,
                                         int amplificationFactor,
                                         OffsetRecord offsetRecord,
                                         boolean hybrid)
    • Method Detail

      • getPartition

        public int getPartition()
      • getLastVTProduceCallFuture

        public java.util.concurrent.CompletableFuture<java.lang.Void> getLastVTProduceCallFuture()
      • setLastVTProduceCallFuture

        public void setLastVTProduceCallFuture​(java.util.concurrent.CompletableFuture<java.lang.Void> lastVTProduceCallFuture)
      • getUserPartition

        public int getUserPartition()
      • getAmplificationFactor

        public int getAmplificationFactor()
      • setDeferredWrite

        public void setDeferredWrite​(boolean deferredWrite)
      • isDeferredWrite

        public boolean isDeferredWrite()
      • isStarted

        public boolean isStarted()
      • isEndOfPushReceived

        public final boolean isEndOfPushReceived()
      • isWaitingForReplicationLag

        public boolean isWaitingForReplicationLag()
      • lagHasCaughtUp

        public void lagHasCaughtUp()
      • hasLagCaughtUp

        public boolean hasLagCaughtUp()
      • isNearlineMetricsRecordingValid

        public boolean isNearlineMetricsRecordingValid​(long producerTimeStampInMs)
        check to ignore calculating latency from pubsub broker to ready to serve for messages that are getting caught up from previous pushes.
        Parameters:
        producerTimeStampInMs - timestamp of the message
      • isCompletionReported

        public boolean isCompletionReported()
      • completionReported

        public void completionReported()
      • isSubscribed

        public boolean isSubscribed()
      • unsubscribe

        public void unsubscribe()
      • isLatchReleased

        public boolean isLatchReleased()
      • releaseLatch

        public void releaseLatch()
      • errorReported

        public void errorReported()
      • isErrorReported

        public boolean isErrorReported()
      • isComplete

        public boolean isComplete()
      • isHybrid

        public final boolean isHybrid()
      • isBatchOnly

        public boolean isBatchOnly()
      • toString

        public java.lang.String toString()
        Overrides:
        toString in class java.lang.Object
      • getProcessedRecordSizeSinceLastSync

        public long getProcessedRecordSizeSinceLastSync()
      • incrementProcessedRecordSizeSinceLastSync

        public void incrementProcessedRecordSizeSinceLastSync​(int recordSize)
      • resetProcessedRecordSizeSinceLastSync

        public void resetProcessedRecordSizeSinceLastSync()
      • setLastLeaderPersistFuture

        public void setLastLeaderPersistFuture​(java.util.concurrent.Future<java.lang.Void> future)
      • getLastLeaderPersistFuture

        public java.util.concurrent.Future<java.lang.Void> getLastLeaderPersistFuture()
      • getLastQueuedRecordPersistedFuture

        public java.util.concurrent.CompletableFuture<java.lang.Void> getLastQueuedRecordPersistedFuture()
      • setLastQueuedRecordPersistedFuture

        public void setLastQueuedRecordPersistedFuture​(java.util.concurrent.CompletableFuture<java.lang.Void> lastQueuedRecordPersistedFuture)
      • setTopicSwitch

        public void setTopicSwitch​(TopicSwitchWrapper topicSwitch)
        Update the in-memory state for TopicSwitch whenever encounter a new TopicSwitch message or after a restart.
      • setConsumeRemotely

        public void setConsumeRemotely​(boolean isConsumingRemotely)
      • consumeRemotely

        public boolean consumeRemotely()
      • initializeExpectedChecksum

        public void initializeExpectedChecksum()
      • finalizeExpectedChecksum

        public void finalizeExpectedChecksum()
      • maybeUpdateExpectedChecksum

        public void maybeUpdateExpectedChecksum​(byte[] key,
                                                Put put)
        Keep updating the checksum for key/value pair received from kafka PUT message. If the checksum instance is not configured via initializeExpectedChecksum() then do nothing. This api will keep the caller's code clean.
        Parameters:
        key -
        put -
      • resetExpectedChecksum

        public void resetExpectedChecksum()
      • getExpectedChecksum

        public byte[] getExpectedChecksum()
      • getLatestMessageConsumedTimestampInMs

        public long getLatestMessageConsumedTimestampInMs()
      • setLatestMessageConsumedTimestampInMs

        public void setLatestMessageConsumedTimestampInMs​(long consumedTimestampInMs)
      • getLatestPolledMessageTimestampInMs

        public long getLatestPolledMessageTimestampInMs()
      • setLatestPolledMessageTimestampInMs

        public void setLatestPolledMessageTimestampInMs​(long timestampInMs)
      • getConsumptionStartTimeInMs

        public long getConsumptionStartTimeInMs()
      • setTransientRecord

        public void setTransientRecord​(int kafkaClusterId,
                                       long kafkaConsumedOffset,
                                       byte[] key,
                                       int valueSchemaId,
                                       org.apache.avro.generic.GenericRecord replicationMetadataRecord)
      • setTransientRecord

        public void setTransientRecord​(int kafkaClusterId,
                                       long kafkaConsumedOffset,
                                       byte[] key,
                                       byte[] value,
                                       int valueOffset,
                                       int valueLen,
                                       int valueSchemaId,
                                       org.apache.avro.generic.GenericRecord replicationMetadataRecord)
      • mayRemoveTransientRecord

        public PartitionConsumptionState.TransientRecord mayRemoveTransientRecord​(int kafkaClusterId,
                                                                                  long kafkaConsumedOffset,
                                                                                  byte[] key)
        This operation is performed atomically to delete the record only when the provided sourceOffset matches.
        Parameters:
        kafkaClusterId -
        kafkaConsumedOffset -
        key -
        Returns:
      • getSourceTopicPartitionNumber

        public int getSourceTopicPartitionNumber​(PubSubTopic topic)
      • getTransientRecordMapSize

        public int getTransientRecordMapSize()
      • recordSubPartitionStatus

        public void recordSubPartitionStatus​(java.lang.String subPartitionStatus)
      • skipKafkaMessage

        public boolean skipKafkaMessage()
      • setSkipKafkaMessage

        public void setSkipKafkaMessage​(boolean skipKafkaMessage)
      • updateLeaderConsumedUpstreamRTOffset

        public void updateLeaderConsumedUpstreamRTOffset​(java.lang.String kafkaUrl,
                                                         long offset)
      • getLeaderConsumedUpstreamRTOffset

        public long getLeaderConsumedUpstreamRTOffset​(java.lang.String kafkaUrl)
      • updateLatestProcessedUpstreamRTOffset

        public void updateLatestProcessedUpstreamRTOffset​(java.lang.String kafkaUrl,
                                                          long offset)
      • updateLatestRTOffsetTriedToProduceToVTMap

        public void updateLatestRTOffsetTriedToProduceToVTMap​(java.lang.String kafkaUrl,
                                                              long offset)
      • getLatestRTOffsetTriedToProduceToVTMap

        public long getLatestRTOffsetTriedToProduceToVTMap​(java.lang.String kafkaUrl)
      • updateLatestIgnoredUpstreamRTOffset

        public void updateLatestIgnoredUpstreamRTOffset​(java.lang.String kafkaUrl,
                                                        long offset)
      • getLatestIgnoredUpstreamRTOffset

        public long getLatestIgnoredUpstreamRTOffset​(java.lang.String kafkaUrl)
      • getLatestProcessedUpstreamRTOffsetWithIgnoredMessages

        public long getLatestProcessedUpstreamRTOffsetWithIgnoredMessages​(java.lang.String kafkaUrl)
      • getLatestProcessedUpstreamRTOffset

        public long getLatestProcessedUpstreamRTOffset​(java.lang.String kafkaUrl)
      • getLatestProcessedUpstreamRTOffsetWithNoDefault

        public java.lang.Long getLatestProcessedUpstreamRTOffsetWithNoDefault​(java.lang.String kafkaUrl)
      • getLeaderOffset

        public long getLeaderOffset​(java.lang.String kafkaURL,
                                    PubSubTopicRepository pubSubTopicRepository)
        The caller of this API should be interested in which offset currently leader should consume from now. 1. If currently leader should consume from real-time topic, return upstream RT offset; 2. if currently leader should consume from version topic, return either remote VT offset or local VT offset, depending on whether the remote consumption flag is on.
      • setStartOfPushTimestamp

        public void setStartOfPushTimestamp​(long startOfPushTimestamp)
      • getStartOfPushTimestamp

        public long getStartOfPushTimestamp()
      • setEndOfPushTimestamp

        public void setEndOfPushTimestamp​(long endOfPushTimestamp)
      • getEndOfPushTimestamp

        public long getEndOfPushTimestamp()
      • updateLatestProcessedLocalVersionTopicOffset

        public void updateLatestProcessedLocalVersionTopicOffset​(long offset)
      • getLatestProcessedLocalVersionTopicOffset

        public long getLatestProcessedLocalVersionTopicOffset()
      • updateLatestProcessedUpstreamVersionTopicOffset

        public void updateLatestProcessedUpstreamVersionTopicOffset​(long offset)
      • getLatestProcessedUpstreamVersionTopicOffset

        public long getLatestProcessedUpstreamVersionTopicOffset()
      • setDataRecoveryCompleted

        public void setDataRecoveryCompleted​(boolean dataRecoveryCompleted)
      • isDataRecoveryCompleted

        public boolean isDataRecoveryCompleted()
      • getLatestProcessedUpstreamRTOffsetMap

        public java.util.Map<java.lang.String,​java.lang.Long> getLatestProcessedUpstreamRTOffsetMap()
      • getLeaderGUID

        public GUID getLeaderGUID()
      • setLeaderGUID

        public void setLeaderGUID​(GUID leaderGUID)
      • getLeaderHostId

        public java.lang.String getLeaderHostId()
      • setLeaderHostId

        public void setLeaderHostId​(java.lang.String hostId)
      • isLeaderCompleted

        public boolean isLeaderCompleted()
      • setLeaderCompleteState

        public void setLeaderCompleteState​(LeaderCompleteState leaderCompleteState)
      • getLastLeaderCompleteStateUpdateInMs

        public long getLastLeaderCompleteStateUpdateInMs()
      • setLastLeaderCompleteStateUpdateInMs

        public void setLastLeaderCompleteStateUpdateInMs​(long lastLeaderCompleteStateUpdateInMs)
      • isFirstHeartBeatSOSReceived

        public boolean isFirstHeartBeatSOSReceived()
      • setFirstHeartBeatSOSReceived

        public void setFirstHeartBeatSOSReceived​(boolean firstHeartBeatSOSReceived)