Package com.linkedin.davinci.stats

Class IngestionStats

  • public class IngestionStats
extends java.lang.Object
    This class contains stats for store ingestion. The stat class is used in VeniceVersionedStats to serve for a single store version or total of all store versions. This class does not contain reporting logic as reporting is done by the IngestionStatsReporter.

    • Field Detail

      • INGESTION_TASK_ERROR_GAUGE

        protected static final java.lang.String INGESTION_TASK_ERROR_GAUGE
        See Also:
        Constant Field Values

      • INGESTION_TASK_PUSH_TIMEOUT_GAUGE

        protected static final java.lang.String INGESTION_TASK_PUSH_TIMEOUT_GAUGE
        See Also:
        Constant Field Values

      • WRITE_COMPUTE_OPERATION_FAILURE

        protected static final java.lang.String WRITE_COMPUTE_OPERATION_FAILURE
        See Also:
        Constant Field Values

      • FOLLOWER_OFFSET_LAG

        protected static final java.lang.String FOLLOWER_OFFSET_LAG
        See Also:
        Constant Field Values

      • LEADER_OFFSET_LAG

        protected static final java.lang.String LEADER_OFFSET_LAG
        See Also:
        Constant Field Values

      • HYBRID_LEADER_OFFSET_LAG

        protected static final java.lang.String HYBRID_LEADER_OFFSET_LAG
        See Also:
        Constant Field Values

      • HYBRID_FOLLOWER_OFFSET_LAG

        protected static final java.lang.String HYBRID_FOLLOWER_OFFSET_LAG
        See Also:
        Constant Field Values

      • BATCH_REPLICATION_LAG

        protected static final java.lang.String BATCH_REPLICATION_LAG
        See Also:
        Constant Field Values

      • BATCH_LEADER_OFFSET_LAG

        protected static final java.lang.String BATCH_LEADER_OFFSET_LAG
        See Also:
        Constant Field Values

      • BATCH_FOLLOWER_OFFSET_LAG

        protected static final java.lang.String BATCH_FOLLOWER_OFFSET_LAG
        See Also:
        Constant Field Values

      • RECORDS_CONSUMED_METRIC_NAME

        protected static final java.lang.String RECORDS_CONSUMED_METRIC_NAME
        See Also:
        Constant Field Values

      • BYTES_CONSUMED_METRIC_NAME

        protected static final java.lang.String BYTES_CONSUMED_METRIC_NAME
        See Also:
        Constant Field Values

      • LEADER_RECORDS_CONSUMED_METRIC_NAME

        protected static final java.lang.String LEADER_RECORDS_CONSUMED_METRIC_NAME
        See Also:
        Constant Field Values

      • LEADER_BYTES_CONSUMED_METRIC_NAME

        protected static final java.lang.String LEADER_BYTES_CONSUMED_METRIC_NAME
        See Also:
        Constant Field Values

      • LEADER_STALLED_HYBRID_INGESTION_METRIC_NAME

        protected static final java.lang.String LEADER_STALLED_HYBRID_INGESTION_METRIC_NAME
        See Also:
        Constant Field Values

      • FOLLOWER_RECORDS_CONSUMED_METRIC_NAME

        protected static final java.lang.String FOLLOWER_RECORDS_CONSUMED_METRIC_NAME
        See Also:
        Constant Field Values

      • FOLLOWER_BYTES_CONSUMED_METRIC_NAME

        protected static final java.lang.String FOLLOWER_BYTES_CONSUMED_METRIC_NAME
        See Also:
        Constant Field Values

      • LEADER_RECORDS_PRODUCED_METRIC_NAME

        protected static final java.lang.String LEADER_RECORDS_PRODUCED_METRIC_NAME
        See Also:
        Constant Field Values

      • LEADER_BYTES_PRODUCED_METRIC_NAME

        protected static final java.lang.String LEADER_BYTES_PRODUCED_METRIC_NAME
        See Also:
        Constant Field Values

      • SUBSCRIBE_ACTION_PREP_LATENCY

        protected static final java.lang.String SUBSCRIBE_ACTION_PREP_LATENCY
        See Also:
        Constant Field Values

      • CONSUMED_RECORD_END_TO_END_PROCESSING_LATENCY

        protected static final java.lang.String CONSUMED_RECORD_END_TO_END_PROCESSING_LATENCY
        See Also:
        Constant Field Values

      • UPDATE_IGNORED_DCR

        protected static final java.lang.String UPDATE_IGNORED_DCR
        See Also:
        Constant Field Values

      • TIMESTAMP_REGRESSION_DCR_ERROR

        protected static final java.lang.String TIMESTAMP_REGRESSION_DCR_ERROR
        See Also:
        Constant Field Values

      • OFFSET_REGRESSION_DCR_ERROR

        protected static final java.lang.String OFFSET_REGRESSION_DCR_ERROR
        See Also:
        Constant Field Values

      • TOMBSTONE_CREATION_DCR

        protected static final java.lang.String TOMBSTONE_CREATION_DCR
        See Also:
        Constant Field Values

      • READY_TO_SERVE_WITH_RT_LAG_METRIC_NAME

        protected static final java.lang.String READY_TO_SERVE_WITH_RT_LAG_METRIC_NAME
        See Also:
        Constant Field Values

      • VERSION_TOPIC_END_OFFSET_REWIND_COUNT

        public static final java.lang.String VERSION_TOPIC_END_OFFSET_REWIND_COUNT
        See Also:
        Constant Field Values

      • TRANSFORMER_ERROR_COUNT

        protected static final java.lang.String TRANSFORMER_ERROR_COUNT
        See Also:
        Constant Field Values

      • NEARLINE_PRODUCER_TO_LOCAL_BROKER_LATENCY

        public static final java.lang.String NEARLINE_PRODUCER_TO_LOCAL_BROKER_LATENCY
        See Also:
        Constant Field Values

      • NEARLINE_LOCAL_BROKER_TO_READY_TO_SERVE_LATENCY

        public static final java.lang.String NEARLINE_LOCAL_BROKER_TO_READY_TO_SERVE_LATENCY
        See Also:
        Constant Field Values

      • TRANSFORMER_LATENCY

        public static final java.lang.String TRANSFORMER_LATENCY
        See Also:
        Constant Field Values

      • TRANSFORMER_LIFECYCLE_START_LATENCY

        public static final java.lang.String TRANSFORMER_LIFECYCLE_START_LATENCY
        See Also:
        Constant Field Values

      • TRANSFORMER_LIFECYCLE_END_LATENCY

        public static final java.lang.String TRANSFORMER_LIFECYCLE_END_LATENCY
        See Also:
        Constant Field Values

      • PRODUCER_CALLBACK_LATENCY

        public static final java.lang.String PRODUCER_CALLBACK_LATENCY
        See Also:
        Constant Field Values

      • LEADER_PREPROCESSING_LATENCY

        public static final java.lang.String LEADER_PREPROCESSING_LATENCY
        See Also:
        Constant Field Values

      • INTERNAL_PREPROCESSING_LATENCY

        public static final java.lang.String INTERNAL_PREPROCESSING_LATENCY
        See Also:
        Constant Field Values

      • BATCH_PROCESSING_REQUEST

        public static final java.lang.String BATCH_PROCESSING_REQUEST
        See Also:
        Constant Field Values

      • BATCH_PROCESSING_REQUEST_SIZE

        public static final java.lang.String BATCH_PROCESSING_REQUEST_SIZE
        See Also:
        Constant Field Values

      • BATCH_PROCESSING_REQUEST_RECORDS

        public static final java.lang.String BATCH_PROCESSING_REQUEST_RECORDS
        See Also:
        Constant Field Values

      • BATCH_PROCESSING_REQUEST_LATENCY

        public static final java.lang.String BATCH_PROCESSING_REQUEST_LATENCY
        See Also:
        Constant Field Values

      • BATCH_PROCESSING_REQUEST_ERROR

        public static final java.lang.String BATCH_PROCESSING_REQUEST_ERROR
        See Also:
        Constant Field Values

    • Method Detail

      • getIngestionTaskErroredGauge

        public int getIngestionTaskErroredGauge()

      • getBatchReplicationLag

        public long getBatchReplicationLag()

      • getLeaderOffsetLag

        public long getLeaderOffsetLag()

      • getBatchLeaderOffsetLag

        public long getBatchLeaderOffsetLag()

      • getHybridLeaderOffsetLag

        public long getHybridLeaderOffsetLag()

      • getFollowerOffsetLag

        public long getFollowerOffsetLag()
        Returns:
        This stats is usually aggregated across the nodes so that we can see the overall lags between leaders and followers. we return 0 instead of StatsErrorCode.INACTIVE_STORE_INGESTION_TASK so the negative error code will not mess up the aggregation.

      • getBatchFollowerOffsetLag

        public long getBatchFollowerOffsetLag()

      • getHybridFollowerOffsetLag

        public long getHybridFollowerOffsetLag()

      • getRegionHybridOffsetLag

        public long getRegionHybridOffsetLag​(int regionId)

      • getWriteComputeErrorCode

        public int getWriteComputeErrorCode()

      • getLeaderStalledHybridIngestion

        public double getLeaderStalledHybridIngestion()
        Returns:
        1 if the leader offset lag is greater than 0 and not actively ingesting data, otherwise 0.

      • getReadyToServeWithRTLag

        public double getReadyToServeWithRTLag()

      • getSubscribePrepLatencyAvg

        public double getSubscribePrepLatencyAvg()

      • getSubscribePrepLatencyMax

        public double getSubscribePrepLatencyMax()

      • recordSubscribePrepLatency

        public void recordSubscribePrepLatency​(double value,
                                       long currentTimeMs)

      • getProducerCallBackLatencyMax

        public double getProducerCallBackLatencyMax()

      • recordProducerCallBackLatency

        public void recordProducerCallBackLatency​(double value,
                                          long currentTimeMs)

      • getLeaderPreprocessingLatencyMax

        public double getLeaderPreprocessingLatencyMax()

      • getLeaderPreprocessingLatencyAvg

        public double getLeaderPreprocessingLatencyAvg()

      • recordLeaderPreprocessingLatency

        public void recordLeaderPreprocessingLatency​(double value,
                                             long currentTimeMs)

      • getInternalPreprocessingLatencyAvg

        public double getInternalPreprocessingLatencyAvg()

      • getInternalPreprocessingLatencyMax

        public double getInternalPreprocessingLatencyMax()

      • recordInternalPreprocessingLatency

        public void recordInternalPreprocessingLatency​(double value,
                                               long currentTimeMs)

      • recordVersionTopicEndOffsetRewind

        public void recordVersionTopicEndOffsetRewind()

      • getVersionTopicEndOffsetRewindCount

        public double getVersionTopicEndOffsetRewindCount()

      • getConsumedRecordEndToEndProcessingLatencyAvg

        public double getConsumedRecordEndToEndProcessingLatencyAvg()

      • getConsumedRecordEndToEndProcessingLatencyMax

        public double getConsumedRecordEndToEndProcessingLatencyMax()

      • recordConsumedRecordEndToEndProcessingLatency

        public void recordConsumedRecordEndToEndProcessingLatency​(double value,
                                                          long currentTimeMs)

      • getRecordsConsumed

        public double getRecordsConsumed()

      • recordRecordsConsumed

        public void recordRecordsConsumed()

      • getBytesConsumed

        public double getBytesConsumed()

      • recordBytesConsumed

        public void recordBytesConsumed​(long value)

      • getLeaderRecordsConsumed

        public double getLeaderRecordsConsumed()

      • recordLeaderRecordsConsumed

        public void recordLeaderRecordsConsumed()

      • getLeaderBytesConsumed

        public double getLeaderBytesConsumed()

      • recordLeaderBytesConsumed

        public void recordLeaderBytesConsumed​(long value)

      • getFollowerRecordsConsumed

        public double getFollowerRecordsConsumed()

      • recordFollowerRecordsConsumed

        public void recordFollowerRecordsConsumed()

      • getFollowerBytesConsumed

        public double getFollowerBytesConsumed()

      • recordFollowerBytesConsumed

        public void recordFollowerBytesConsumed​(long value)

      • recordUpdateIgnoredDCR

        public void recordUpdateIgnoredDCR()

      • recordTotalDCR

        public void recordTotalDCR()

      • recordTimestampRegressionDCRError

        public void recordTimestampRegressionDCRError()

      • recordOffsetRegressionDCRError

        public void recordOffsetRegressionDCRError()

      • recordTombStoneCreationDCR

        public void recordTombStoneCreationDCR()

      • getRegionHybridBytesConsumed

        public double getRegionHybridBytesConsumed​(int regionId)

      • recordRegionHybridBytesConsumed

        public void recordRegionHybridBytesConsumed​(int regionId,
                                            double value,
                                            long currentTimeMs)

      • getRegionHybridRecordsConsumed

        public double getRegionHybridRecordsConsumed​(int regionId)

      • recordRegionHybridRecordsConsumed

        public void recordRegionHybridRecordsConsumed​(int regionId,
                                              double value,
                                              long currentTimeMs)

      • getRegionHybridAvgConsumedOffset

        public double getRegionHybridAvgConsumedOffset​(int regionId)

      • recordRegionHybridAvgConsumedOffset

        public void recordRegionHybridAvgConsumedOffset​(int regionId,
                                                double value,
                                                long currentTimeMs)

      • getLeaderRecordsProduced

        public double getLeaderRecordsProduced()

      • getUpdateIgnoredRate

        public double getUpdateIgnoredRate()

      • getTotalDCRRate

        public double getTotalDCRRate()

      • getTombstoneCreationDCRRate

        public double getTombstoneCreationDCRRate()

      • getTimestampRegressionDCRRate

        public double getTimestampRegressionDCRRate()

      • getOffsetRegressionDCRRate

        public double getOffsetRegressionDCRRate()

      • recordLeaderRecordsProduced

        public void recordLeaderRecordsProduced​(long value)

      • getLeaderBytesProduced

        public double getLeaderBytesProduced()

      • recordLeaderBytesProduced

        public void recordLeaderBytesProduced​(long value)

      • setIngestionTaskPushTimeoutGauge

        public void setIngestionTaskPushTimeoutGauge​(int value)

      • getIngestionTaskPushTimeoutGauge

        public int getIngestionTaskPushTimeoutGauge()

      • getNearlineProducerToLocalBrokerLatencyAvg

        public double getNearlineProducerToLocalBrokerLatencyAvg()

      • getNearlineProducerToLocalBrokerLatencyMax

        public double getNearlineProducerToLocalBrokerLatencyMax()

      • getNearlineLocalBrokerToReadyToServeLatencyAvg

        public double getNearlineLocalBrokerToReadyToServeLatencyAvg()

      • getNearlineLocalBrokerToReadyToServeLatencyMax

        public double getNearlineLocalBrokerToReadyToServeLatencyMax()

      • recordNearlineProducerToLocalBrokerLatency

        public void recordNearlineProducerToLocalBrokerLatency​(double value,
                                                       long currentTimeMs)

      • recordNearlineLocalBrokerToReadyToServeLatency

        public void recordNearlineLocalBrokerToReadyToServeLatency​(double value,
                                                           long currentTimeMs)

      • recordTransformerError

        public void recordTransformerError​(double value,
                                   long currentTimeMs)

      • registerTransformerErrorSensor

        public void registerTransformerErrorSensor()

      • getTransformerErrorCount

        public double getTransformerErrorCount()

      • recordTransformerLatency

        public void recordTransformerLatency​(double value,
                                     long currentTimeMs)

      • registerTransformerLatencySensor

        public void registerTransformerLatencySensor()

      • recordTransformerLifecycleStartLatency

        public void recordTransformerLifecycleStartLatency​(double value,
                                                   long currentTimeMs)

      • registerTransformerLifecycleStartLatencySensor

        public void registerTransformerLifecycleStartLatencySensor()

      • recordTransformerLifecycleEndLatency

        public void recordTransformerLifecycleEndLatency​(double value,
                                                 long currentTimeMs)

      • registerTransformerLifecycleEndLatencySensor

        public void registerTransformerLifecycleEndLatencySensor()

      • recordIdleTime

        public void recordIdleTime​(long value)

      • getIdleTime

        public double getIdleTime()

      • recordProducerSourceBrokerLatencyMs

        public void recordProducerSourceBrokerLatencyMs​(double value,
                                                long currentTimeMs)

      • recordSourceBrokerLeaderConsumerLatencyMs

        public void recordSourceBrokerLeaderConsumerLatencyMs​(double value,
                                                      long currentTimeMs)

      • getSourceBrokerLeaderConsumerLatencySensor

        public WritePathLatencySensor getSourceBrokerLeaderConsumerLatencySensor()

      • recordProducerLocalBrokerLatencyMs

        public void recordProducerLocalBrokerLatencyMs​(double value,
                                               long currentTimeMs)

      • recordLocalBrokerFollowerConsumerLatencyMs

        public void recordLocalBrokerFollowerConsumerLatencyMs​(double value,
                                                       long currentTimeMs)

      • getLocalBrokerFollowerConsumerLatencySensor

        public WritePathLatencySensor getLocalBrokerFollowerConsumerLatencySensor()

      • recordLeaderProducerCompletionLatencyMs

        public void recordLeaderProducerCompletionLatencyMs​(double value,
                                                    long currentTimeMs)

      • getLeaderProducerCompletionLatencySensor

        public WritePathLatencySensor getLeaderProducerCompletionLatencySensor()

      • recordBatchProcessingRequest

        public void recordBatchProcessingRequest​(int size,
                                         long currentTimeMs)

      • getBatchProcessingRequest

        public double getBatchProcessingRequest()

      • getBatchProcessingRequestRecords

        public double getBatchProcessingRequestRecords()

      • recordBatchProcessingRequestError

        public void recordBatchProcessingRequestError()

      • getBatchProcessingRequestError

        public double getBatchProcessingRequestError()

      • recordBatchProcessingRequestLatency

        public void recordBatchProcessingRequestLatency​(double latency,
                                                long currentTimeMs)

      • getBatchProcessingRequestLatencySensor

        public WritePathLatencySensor getBatchProcessingRequestLatencySensor()

      • unAvailableToZero

        public static double unAvailableToZero​(double value)