Package com.linkedin.venice.hadoop

Class PushJobSetting

java.lang.Object
com.linkedin.venice.hadoop.PushJobSetting
All Implemented Interfaces:
Serializable
public class PushJobSetting extends Object implements Serializable
This class carries the state for the duration of the VenicePushJob. Consider making breaking changes carefully.
See Also:

  • Field Details

    • jobStartTimeMs

      public long jobStartTimeMs

    • jobId

      public String jobId

    • jobExecutionId

      public String jobExecutionId

    • jobServerName

      public String jobServerName

    • sharedTmpDir

      public String sharedTmpDir

    • jobTmpDir

      public String jobTmpDir

    • enableSSL

      public boolean enableSSL

    • vpjEntryClass

      public Class<? extends VenicePushJob> vpjEntryClass

    • veniceControllerUrl

      public String veniceControllerUrl

    • storeName

      public String storeName

    • inputURI

      public String inputURI

    • sourceGridFabric

      public String sourceGridFabric

    • batchNumBytes

      public int batchNumBytes

    • isIncrementalPush

      public boolean isIncrementalPush

    • incrementalPushVersion

      public String incrementalPushVersion

    • isDuplicateKeyAllowed

      public boolean isDuplicateKeyAllowed

    • controllerRetries

      public int controllerRetries

    • controllerStatusPollRetries

      public int controllerStatusPollRetries

    • pollJobStatusIntervalMs

      public long pollJobStatusIntervalMs

    • jobStatusInUnknownStateTimeoutMs

      public long jobStatusInUnknownStateTimeoutMs

    • pushJobTimeoutOverrideMs

      public long pushJobTimeoutOverrideMs

    • sendControlMessagesDirectly

      public boolean sendControlMessagesDirectly

    • isSourceETL

      public boolean isSourceETL

    • enableWriteCompute

      public boolean enableWriteCompute

    • isSourceKafka

      public boolean isSourceKafka

    • repushSourcePubsubBroker

      public String repushSourcePubsubBroker
      Broker URL for consuming/reading existing version data during a KIF (Kafka Input Format) repush.

      This is the "input/source" side of a repush: the Kafka broker from which the previous version's data is read. It is set from one of two sources:

      1. RepushInfoResponse returned by the controller (which resolves the fabric name from KAFKA_INPUT_FABRIC to a broker URL), or
      2. An explicit VENICE_REPUSH_SOURCE_PUBSUB_BROKER property provided by the caller.

      This may point to a different fabric than pushDestinationPubsubBroker when the repush input fabric differs from the NR source fabric. For example, a repush may read v1 data from dc-1 but write v2 data to dc-0 (the NR source).

      See Also:

    • kafkaInputTopic

      public String kafkaInputTopic

    • repushSourceVersion

      public int repushSourceVersion

    • rewindTimeInSecondsOverride

      public long rewindTimeInSecondsOverride

    • pushToSeparateRealtimeTopicEnabled

      public boolean pushToSeparateRealtimeTopicEnabled

    • versionSeparateRealTimeTopicEnabled

      public boolean versionSeparateRealTimeTopicEnabled

    • kafkaInputCombinerEnabled

      public boolean kafkaInputCombinerEnabled

    • kafkaInputBuildNewDictEnabled

      public boolean kafkaInputBuildNewDictEnabled

    • validateRemoteReplayPolicy

      public BufferReplayPolicy validateRemoteReplayPolicy

    • suppressEndOfPushMessage

      public boolean suppressEndOfPushMessage

    • deferVersionSwap

      public boolean deferVersionSwap

    • extendedSchemaValidityCheckEnabled

      public boolean extendedSchemaValidityCheckEnabled

    • compressionMetricCollectionEnabled

      public boolean compressionMetricCollectionEnabled
      Refer VenicePushJobConstants.COMPRESSION_METRIC_COLLECTION_ENABLED

    • repushTTLEnabled

      public boolean repushTTLEnabled

    • isCompliancePush

      public boolean isCompliancePush

    • repushTTLStartTimeMs

      public long repushTTLStartTimeMs

    • rmdSchemaDir

      public String rmdSchemaDir

    • valueSchemaDir

      public String valueSchemaDir

    • controllerD2ServiceName

      public String controllerD2ServiceName

    • parentControllerRegionD2ZkHosts

      public String parentControllerRegionD2ZkHosts

    • childControllerRegionD2ZkHosts

      public String childControllerRegionD2ZkHosts

    • livenessHeartbeatEnabled

      public boolean livenessHeartbeatEnabled

    • livenessHeartbeatStoreName

      public String livenessHeartbeatStoreName

    • multiRegion

      public boolean multiRegion

    • d2Routing

      public boolean d2Routing

    • targetedRegions

      public String targetedRegions

    • isTargetedRegionPushEnabled

      public boolean isTargetedRegionPushEnabled

    • isTargetRegionPushWithDeferredSwapEnabled

      public boolean isTargetRegionPushWithDeferredSwapEnabled

    • targetRegionPushWithDeferredSwapWaitTime

      public int targetRegionPushWithDeferredSwapWaitTime

    • isDegradedModePush

      public boolean isDegradedModePush

    • degradedDatacenters

      public Set<String> degradedDatacenters

    • isSystemSchemaReaderEnabled

      public boolean isSystemSchemaReaderEnabled

    • isZstdDictCreationRequired

      public boolean isZstdDictCreationRequired

    • isZstdDictCreationSuccess

      public boolean isZstdDictCreationSuccess

    • dataWriterComputeJobClass

      public Class<? extends DataWriterComputeJob> dataWriterComputeJobClass

    • clusterName

      public String clusterName

    • storeKeySchema

      public org.apache.avro.Schema storeKeySchema

    • isChunkingEnabled

      public boolean isChunkingEnabled

    • isRmdChunkingEnabled

      public boolean isRmdChunkingEnabled

    • storeStorageQuota

      public long storeStorageQuota

    • targetStorageMode

      public StorageMode targetStorageMode
      Storage mode of the new version being pushed to, read from Version.getStorageMode() after the controller creates the new version. The version's storageMode is fixed at creation time (the controller copies the store-level value onto the version per linkedin/venice#2823), so reading the version's value rather than the store-level value avoids a race where a concurrent UpdateStore mutates the store-level value between job setup and version creation. Only populated when the VPJ-side dual-write writer-class is configured; otherwise stays StorageMode.INTERNAL (the default).

    • isSchemaAutoRegisterFromPushJobEnabled

      public boolean isSchemaAutoRegisterFromPushJobEnabled

    • storeCompressionStrategy

      public CompressionStrategy storeCompressionStrategy

    • isStoreWriteComputeEnabled

      public boolean isStoreWriteComputeEnabled

    • isStoreIncrementalPushEnabled

      public boolean isStoreIncrementalPushEnabled

    • hybridStoreConfig

      public transient HybridStoreConfig hybridStoreConfig

    • storeResponse

      public transient StoreResponse storeResponse

    • topic

      public String topic

    • version

      public int version
      Version part of the store-version / topic name

    • partitionCount

      public int partitionCount

    • pushDestinationPubsubBroker

      public String pushDestinationPubsubBroker
      Broker URL for producing/writing new version topic data.

      This is the "output/destination" side of a push: the Kafka broker to which new version data records are written. It is set from VersionCreationResponse.getKafkaBootstrapServers(), which returns the broker for the NR (Native Replication) source region. In NR mode, data is first written to this broker, then replicated to other regions by the storage nodes.

      For a cross-fabric repush (where the input fabric differs from the NR source), this URL should point to the NR source fabric's broker — not the input fabric. For example, if NR source = dc-0 and repush reads from dc-1, this URL should be dc-0's broker.

      See Also:

    • sslToKafka

      public boolean sslToKafka

    • topicCompressionStrategy

      public CompressionStrategy topicCompressionStrategy

    • partitionerClass

      public String partitionerClass

    • partitionerParams

      public Map<String,String> partitionerParams

    • chunkingEnabled

      public boolean chunkingEnabled

    • rmdChunkingEnabled

      public boolean rmdChunkingEnabled

    • maxRecordSizeBytes

      public int maxRecordSizeBytes

    • enableUncompressedRecordSizeLimit

      public boolean enableUncompressedRecordSizeLimit

    • kafkaSourceRegion

      public String kafkaSourceRegion

    • repushInfoResponse

      public transient RepushInfoResponse repushInfoResponse

    • repushUseFallbackValueSchemaId

      public boolean repushUseFallbackValueSchemaId

    • isAvro

      public boolean isAvro

    • valueSchemaId

      public int valueSchemaId

    • rmdSchemaId

      public int rmdSchemaId

    • derivedSchemaId

      public int derivedSchemaId

    • keyField

      public String keyField

    • valueField

      public String valueField

    • rmdField

      public String rmdField

    • inputDataSchema

      public org.apache.avro.Schema inputDataSchema

    • inputDataSchemaString

      public String inputDataSchemaString

    • keySchema

      public org.apache.avro.Schema keySchema

    • keySchemaString

      public String keySchemaString

    • valueSchema

      public org.apache.avro.Schema valueSchema

    • valueSchemaString

      public String valueSchemaString

    • replicationMetadataSchemaString

      public String replicationMetadataSchemaString

    • vsonInputKeySchema

      public VsonSchema vsonInputKeySchema

    • vsonInputKeySchemaString

      public String vsonInputKeySchemaString

    • vsonInputValueSchema

      public VsonSchema vsonInputValueSchema

    • vsonInputValueSchemaString

      public String vsonInputValueSchemaString

    • generatePartialUpdateRecordFromInput

      public boolean generatePartialUpdateRecordFromInput

    • etlValueSchemaTransformation

      public ETLValueSchemaTransformation etlValueSchemaTransformation

    • newKmeSchemasFromController

      public Map<Integer,String> newKmeSchemasFromController

    • inputHasRecords

      public boolean inputHasRecords

    • inputFileDataSizeInBytes

      public long inputFileDataSizeInBytes

    • sourceKafkaInputVersionInfo

      public transient Version sourceKafkaInputVersionInfo

    • sourceVersionCompressionStrategy

      public CompressionStrategy sourceVersionCompressionStrategy

    • sourceVersionChunkingEnabled

      public boolean sourceVersionChunkingEnabled

    • sourceDictionary

      public byte[] sourceDictionary

    • topicDictionary

      public byte[] topicDictionary

    • materializedViewConfigFlatMap

      public String materializedViewConfigFlatMap

    • isBatchWriteOptimizationForHybridStoreEnabled

      public boolean isBatchWriteOptimizationForHybridStoreEnabled

    • isSortedIngestionEnabled

      public boolean isSortedIngestionEnabled

    • allowRegularPushWithTTLRepush

      public boolean allowRegularPushWithTTLRepush

  • Constructor Details

    • PushJobSetting

      public PushJobSetting()