Package com.linkedin.venice.controllerapi

Class ControllerClient

  • All Implemented Interfaces:
    java.io.Closeable, java.lang.AutoCloseable
    Direct Known Subclasses:
    D2ControllerClient
    public class ControllerClient
extends java.lang.Object
implements java.io.Closeable

    • Constructor Detail

      • ControllerClient

        public ControllerClient​(java.lang.String clusterName,
                        java.lang.String discoveryUrls)

      • ControllerClient

        public ControllerClient​(java.lang.String clusterName,
                        java.lang.String discoveryUrls,
                        java.util.Optional<SSLFactory> sslFactory)
        Parameters:
        discoveryUrls - comma-delimited urls to find leader controller.

    • Method Detail

      • close

        public void close()
        Specified by:
        close in interface java.lang.AutoCloseable
        Specified by:
        close in interface java.io.Closeable

      • discoverLeaderController

        protected java.lang.String discoverLeaderController()

      • getStore

        public StoreResponse getStore​(java.lang.String storeName)

      • getStore

        public StoreResponse getStore​(java.lang.String storeName,
                              int timeoutMs)

      • getRepushInfo

        public RepushInfoResponse getRepushInfo​(java.lang.String storeName,
                                        java.util.Optional<java.lang.String> fabricName)

      • getFutureVersions

        public MultiStoreStatusResponse getFutureVersions​(java.lang.String clusterName,
                                                  java.lang.String storeName)

      • getBackupVersions

        public MultiStoreStatusResponse getBackupVersions​(java.lang.String clusterName,
                                                  java.lang.String storeName)

      • getStore

        @Deprecated
public static StoreResponse getStore​(java.lang.String urlsToFindLeaderController,
                                     java.lang.String clusterName,
                                     java.lang.String storeName)
        Deprecated.

      • getInUseSchemaIds

        public SchemaUsageResponse getInUseSchemaIds​(java.lang.String storeName)

      • deleteValueSchemas

        public ControllerResponse deleteValueSchemas​(java.lang.String storeName,
                                             java.util.List<java.lang.String> schemaIds)

      • requestTopicForWrites

        public VersionCreationResponse requestTopicForWrites​(java.lang.String storeName,
                                                     long storeSize,
                                                     Version.PushType pushType,
                                                     java.lang.String pushJobId,
                                                     boolean sendStartOfPush,
                                                     boolean sorted,
                                                     boolean wcEnabled,
                                                     java.util.Optional<java.lang.String> partitioners,
                                                     java.util.Optional<java.lang.String> compressionDictionary,
                                                     java.util.Optional<java.lang.String> sourceGridFabric,
                                                     boolean batchJobHeartbeatEnabled,
                                                     long rewindTimeInSecondsOverride)

      • requestTopicForWrites

        public VersionCreationResponse requestTopicForWrites​(java.lang.String storeName,
                                                     long storeSize,
                                                     Version.PushType pushType,
                                                     java.lang.String pushJobId,
                                                     boolean sendStartOfPush,
                                                     boolean sorted,
                                                     boolean wcEnabled,
                                                     java.util.Optional<java.lang.String> partitioners,
                                                     java.util.Optional<java.lang.String> compressionDictionary,
                                                     java.util.Optional<java.lang.String> sourceGridFabric,
                                                     boolean batchJobHeartbeatEnabled,
                                                     long rewindTimeInSecondsOverride,
                                                     boolean deferVersionSwap)

      • requestTopicForWrites

        public VersionCreationResponse requestTopicForWrites​(java.lang.String storeName,
                                                     long storeSize,
                                                     Version.PushType pushType,
                                                     java.lang.String pushJobId,
                                                     boolean sendStartOfPush,
                                                     boolean sorted,
                                                     boolean wcEnabled,
                                                     java.util.Optional<java.lang.String> partitioners,
                                                     java.util.Optional<java.lang.String> compressionDictionary,
                                                     java.util.Optional<java.lang.String> sourceGridFabric,
                                                     boolean batchJobHeartbeatEnabled,
                                                     long rewindTimeInSecondsOverride,
                                                     boolean deferVersionSwap,
                                                     java.lang.String targetedRegions,
                                                     int repushSourceVersion)
        Request a topic for the VeniceWriter to write into. A new VPJ push, or a Samza bulk processing job should both use this method. The push job ID needs to be unique for this push. Multiple requests with the same pushJobId are idempotent and will return the same topic.
        Parameters:
        storeName - Name of the store being written to.
        storeSize - Estimated size of push in bytes, used to determine partitioning
        pushJobId - Unique identifier for this job
        sendStartOfPush - Whether controller should send START_OF_PUSH message to the newly created topic, while adding a new version. This is currently used in Samza batch load, a.k.a. grandfather
        sorted - Whether the push is going to contain sorted data (in each partition) or not
        wcEnabled - Whether write compute is enabled for this push job or not
        partitioners - partitioner class names in a string seperated by comma
        compressionDictionary - Base64 encoded dictionary to be used to perform dictionary compression
        sourceGridFabric - An identifier of the data center which is used in native replication to determine the Kafka URL
        batchJobHeartbeatEnabled - whether batch push job enables the heartbeat
        rewindTimeInSecondsOverride - if a valid value is specified (>=0) for hybrid store, this param will override the default store-level rewindTimeInSeconds config.
        deferVersionSwap - whether to defer version swap after the push is done
        targetedRegions - the list of regions that is separated by comma for targeted region push.
        repushSourceVersion -
        Returns:
        VersionCreationResponse includes topic and partitioning

      • addVersionAndStartIngestion

        public VersionResponse addVersionAndStartIngestion​(java.lang.String storeName,
                                                   java.lang.String pushJobId,
                                                   int version,
                                                   int partitionCount,
                                                   Version.PushType pushType,
                                                   java.lang.String remoteKafkaBootstrapServers,
                                                   long rewindTimeInSecondsOverride,
                                                   int replicationMetadataVersionId)
        Used for store migration to add version and start ingestion in destination cluster for new pushes in the source cluster during the ingestion. The idea is like copying or create a version on an existing topic. Different use cases can be explored and expanded in the future. Applicable only to child controllers.
        Parameters:
        storeName - of the original push.
        pushJobId - of the original push.
        version - of the original push.
        partitionCount - of the original push.
        pushType - of the producer.
        remoteKafkaBootstrapServers - of the original push.
        rewindTimeInSecondsOverride - of the original push.
        replicationMetadataVersionId - of the original push.
        Returns:

      • writeEndOfPush

        public ControllerResponse writeEndOfPush​(java.lang.String storeName,
                                         int version)

      • sendEmptyPushAndWait

        public ControllerResponse sendEmptyPushAndWait​(java.lang.String storeName,
                                               java.lang.String pushJobId,
                                               long storeSize,
                                               long timeOut)
        Sends and empty push to the venice controller, but verifies that the push has succeeded before returning to the caller.
        Parameters:
        storeName - the store name for which the empty push is for
        pushJobId - the push job id for the push
        storeSize - the size of the store (currently unused)
        timeOut - max amount of time this function should take before returning in MILLISECONDS. Retries sent to the controller have 2-second sleeps between them. So a timeout should be chosen that is larger, and a multiple of 2 seconds preferably.
        Returns:
        the response from the controller. Either a successful one, or a failed one with more information.

      • createNewStoreWithParameters

        public ControllerResponse createNewStoreWithParameters​(java.lang.String storeName,
                                                       java.lang.String owner,
                                                       java.lang.String keySchema,
                                                       java.lang.String valueSchema,
                                                       UpdateStoreQueryParams updateStoreQueryParams)
        Simplified API that wraps together the store create and update functionalities with some clean up functionality
        Parameters:
        storeName - the store name to create and update
        owner - the owner of this store to be created
        keySchema - Schema of the key for row retrieval for this store
        valueSchema - Schema of the value for rows in this new store
        updateStoreQueryParams - What parameters should be applied to this store after it's creation
        Returns:
        The response from the first failed operation of store creation or modification

      • emptyPush

        public VersionCreationResponse emptyPush​(java.lang.String storeName,
                                         java.lang.String pushJobId,
                                         long storeSize)

      • createNewStore

        public NewStoreResponse createNewStore​(java.lang.String storeName,
                                       java.lang.String owner,
                                       java.lang.String keySchema,
                                       java.lang.String valueSchema)

      • createNewStore

        public NewStoreResponse createNewStore​(java.lang.String storeName,
                                       java.lang.String owner,
                                       java.lang.String keySchema,
                                       java.lang.String valueSchema,
                                       java.lang.String accessPermissions)

      • createNewSystemStore

        public NewStoreResponse createNewSystemStore​(java.lang.String storeName,
                                             java.lang.String owner,
                                             java.lang.String keySchema,
                                             java.lang.String valueSchema)

      • migrateStore

        public StoreMigrationResponse migrateStore​(java.lang.String storeName,
                                           java.lang.String destClusterName)

      • completeMigration

        public StoreMigrationResponse completeMigration​(java.lang.String storeName,
                                                java.lang.String destClusterName)

      • abortMigration

        public StoreMigrationResponse abortMigration​(java.lang.String storeName,
                                             java.lang.String destClusterName)
        This command should be sent to src controller, not dest controller

      • wipeCluster

        public ControllerResponse wipeCluster​(java.lang.String fabric,
                                      java.util.Optional<java.lang.String> storeName,
                                      java.util.Optional<java.lang.Integer> versionNum)

      • compareStore

        public StoreComparisonResponse compareStore​(java.lang.String storeName,
                                            java.lang.String fabricA,
                                            java.lang.String fabricB)

      • copyOverStoreMetadata

        public StoreResponse copyOverStoreMetadata​(java.lang.String sourceFabric,
                                           java.lang.String destFabric,
                                           java.lang.String storeName)

      • disableAndDeleteStore

        public ControllerResponse disableAndDeleteStore​(java.lang.String storeName)

      • overrideSetActiveVersion

        public VersionResponse overrideSetActiveVersion​(java.lang.String storeName,
                                                int version)

      • rollbackToBackupVersion

        public ControllerResponse rollbackToBackupVersion​(java.lang.String storeName,
                                                  java.lang.String regionFilter)

      • rollbackToBackupVersion

        public ControllerResponse rollbackToBackupVersion​(java.lang.String storeName)

      • rollForwardToFutureVersion

        public ControllerResponse rollForwardToFutureVersion​(java.lang.String storeName,
                                                     java.lang.String regionFilter)

      • rollForwardToFutureVersion

        public ControllerResponse rollForwardToFutureVersion​(java.lang.String storeName)

      • killOfflinePushJob

        public ControllerResponse killOfflinePushJob​(java.lang.String kafkaTopic)

      • skipAdminMessage

        public ControllerResponse skipAdminMessage​(java.lang.String offset,
                                           boolean skipDIV)

      • updateKafkaTopicLogCompaction

        public ControllerResponse updateKafkaTopicLogCompaction​(java.lang.String kafkaTopicName,
                                                        boolean logCompactionEnabled)

      • updateKafkaTopicRetention

        public ControllerResponse updateKafkaTopicRetention​(java.lang.String kafkaTopicName,
                                                    long retentionInMs)

      • updateKafkaTopicMinInSyncReplica

        public ControllerResponse updateKafkaTopicMinInSyncReplica​(java.lang.String kafkaTopicName,
                                                           int minISR)

      • retryableRequest

        public <R extends ControllerResponse> R retryableRequest​(int totalAttempts,
                                                         java.util.function.Function<ControllerClient,​R> request,
                                                         java.util.function.Function<R,​java.lang.Boolean> abortRetryCondition)

      • retryableRequest

        public static <C extends ControllerClient,​R extends ControllerResponse> R retryableRequest​(C client,
                                                                                                 int totalAttempts,
                                                                                                 java.util.function.Function<C,​R> request)
        Useful for pieces of code which want to have a test mocking the result of the function that's passed in...

      • retryableRequest

        public static <C extends ControllerClient,​R extends ControllerResponse> R retryableRequest​(C client,
                                                                                                 int totalAttempts,
                                                                                                 java.util.function.Function<C,​R> request,
                                                                                                 java.util.function.Function<R,​java.lang.Boolean> abortRetryCondition)

      • queryOverallJobStatus

        public JobStatusQueryResponse queryOverallJobStatus​(java.lang.String kafkaTopic,
                                                    java.util.Optional<java.lang.String> incrementalPushVersion,
                                                    java.lang.String targetedRegions)
        This method has a longer timeout intended to be used to query the overall job status on a parent controller. The extended timeout is meant for the parent controller to query each colo's child controller for the job status and aggregate the results. Use queryJobStatus(String, Optional) instead if the target is a child controller.
        Parameters:
        kafkaTopic - , the version topic name of the push job.
        incrementalPushVersion - , the optional incremental push version of the push job.
        targetedRegions - , the list of regions that is separated by comma for targeted region push.
        Returns:

      • queryOverallJobStatus

        public JobStatusQueryResponse queryOverallJobStatus​(java.lang.String kafkaTopic,
                                                    java.util.Optional<java.lang.String> incrementalPushVersion)

      • queryJobStatus

        public JobStatusQueryResponse queryJobStatus​(java.lang.String kafkaTopic,
                                             java.util.Optional<java.lang.String> incrementalPushVersion)
        This method is used to query the job status from a controller. It is expected to be a child controller thus a shorter timeout is enforced. Use queryOverallJobStatus(String, Optional) instead if the target is a parent controller.

      • queryJobStatus

        public JobStatusQueryResponse queryJobStatus​(java.lang.String kafkaTopic,
                                             java.util.Optional<java.lang.String> incrementalPushVersion,
                                             java.lang.String targetedRegions)

      • queryJobStatus

        public JobStatusQueryResponse queryJobStatus​(java.lang.String kafkaTopic,
                                             java.util.Optional<java.lang.String> incrementalPushVersion,
                                             int timeoutMs,
                                             java.lang.String targetedRegions)

      • queryDetailedJobStatus

        public JobStatusQueryResponse queryDetailedJobStatus​(java.lang.String kafkaTopic,
                                                     java.lang.String region)
        This method will retrieve detailed job status, including uncompleted partitions and replicas from a child region. If the request is sent to a parent controller, it will be forwarded to a child controller in the specified region. This method is used for data recovery progress monitoring.

      • sendPushJobDetails

        public ControllerResponse sendPushJobDetails​(java.lang.String storeName,
                                             int version,
                                             byte[] pushJobDetails)

      • queryStoreList

        public MultiStoreResponse queryStoreList​(boolean includeSystemStores)

      • queryStoreList

        public MultiStoreResponse queryStoreList​(boolean includeSystemStores,
                                         java.util.Optional<java.lang.String> configNameFilter,
                                         java.util.Optional<java.lang.String> configValueFilter)

      • enableStoreWrites

        public ControllerResponse enableStoreWrites​(java.lang.String storeName,
                                            boolean enable)

      • enableStoreReads

        public ControllerResponse enableStoreReads​(java.lang.String storeName,
                                           boolean enable)

      • enableStoreReadWrites

        public ControllerResponse enableStoreReadWrites​(java.lang.String storeName,
                                                boolean enable)

      • deleteOldVersion

        public VersionResponse deleteOldVersion​(java.lang.String storeName,
                                        int versionNum)

      • isNodeRemovable

        public NodeStatusResponse isNodeRemovable​(java.lang.String instanceId)

      • isNodeRemovable

        public NodeStatusResponse isNodeRemovable​(java.lang.String instanceId,
                                          java.util.List<java.lang.String> lockedNodeIds)

      • addNodeIntoAllowList

        public ControllerResponse addNodeIntoAllowList​(java.lang.String instanceId)

      • removeNodeFromAllowList

        public ControllerResponse removeNodeFromAllowList​(java.lang.String instanceId)

      • removeNodeFromCluster

        public ControllerResponse removeNodeFromCluster​(java.lang.String instanceId)

      • getAggregatedHealthStatus

        public StoppableNodeStatusResponse getAggregatedHealthStatus​(java.lang.String clusterName,
                                                             java.util.List<java.lang.String> instances,
                                                             java.util.List<java.lang.String> toBeStoppedInstances)
                                                      throws com.fasterxml.jackson.core.JsonProcessingException
        Throws:
        com.fasterxml.jackson.core.JsonProcessingException

      • listReplicas

        public MultiReplicaResponse listReplicas​(java.lang.String storeName,
                                         int version)

      • listStorageNodeReplicas

        public MultiReplicaResponse listStorageNodeReplicas​(java.lang.String instanceId)

      • listChildControllers

        public ChildAwareResponse listChildControllers​(java.lang.String clusterName)

      • getKeySchema

        public SchemaResponse getKeySchema​(java.lang.String storeName)

      • addValueSchema

        public SchemaResponse addValueSchema​(java.lang.String storeName,
                                     java.lang.String valueSchemaStr)

      • addValueSchema

        public SchemaResponse addValueSchema​(java.lang.String storeName,
                                     java.lang.String valueSchemaStr,
                                     int valueSchemaId)

      • addDerivedSchema

        public SchemaResponse addDerivedSchema​(java.lang.String storeName,
                                       int valueSchemaId,
                                       java.lang.String derivedSchemaStr)

      • addDerivedSchema

        public SchemaResponse addDerivedSchema​(java.lang.String storeName,
                                       int valueSchemaId,
                                       java.lang.String derivedSchemaStr,
                                       int derivedSchemaId)

      • removeDerivedSchema

        public SchemaResponse removeDerivedSchema​(java.lang.String storeName,
                                          int valueSchemaId,
                                          int derivedSchemaId)

      • setStorePartitionCount

        public PartitionResponse setStorePartitionCount​(java.lang.String storeName,
                                                java.lang.String partitionNum)

      • setStoreOwner

        public OwnerResponse setStoreOwner​(java.lang.String storeName,
                                   java.lang.String owner)

      • getValueSchema

        public SchemaResponse getValueSchema​(java.lang.String storeName,
                                     int valueSchemaId)

      • getValueSchemaID

        public SchemaResponse getValueSchemaID​(java.lang.String storeName,
                                       java.lang.String valueSchemaStr)

      • getValueOrDerivedSchemaId

        public SchemaResponse getValueOrDerivedSchemaId​(java.lang.String storeName,
                                                java.lang.String derivedSchemaStr)

      • getAllValueSchema

        public MultiSchemaResponse getAllValueSchema​(java.lang.String storeName)

      • getAllReplicationMetadataSchemas

        public MultiSchemaResponse getAllReplicationMetadataSchemas​(java.lang.String storeName)

      • getAllValueAndDerivedSchema

        public MultiSchemaResponse getAllValueAndDerivedSchema​(java.lang.String storeName)

      • enableThrottling

        public ControllerResponse enableThrottling​(boolean isThrottlingEnabled)

      • enableMaxCapacityProtection

        public ControllerResponse enableMaxCapacityProtection​(boolean isMaxCapacityProtection)

      • enableQuotaRebalanced

        public ControllerResponse enableQuotaRebalanced​(boolean isQuotaRebalanced,
                                                int expectRouterCount)

      • setMigrationPushStrategy

        public ControllerResponse setMigrationPushStrategy​(java.lang.String voldemortStoreName,
                                                   java.lang.String pushStrategy)

      • updateAclForStore

        public AclResponse updateAclForStore​(java.lang.String storeName,
                                     java.lang.String accessPermissions)

      • getAclForStore

        public AclResponse getAclForStore​(java.lang.String storeName)

      • deleteAclForStore

        public AclResponse deleteAclForStore​(java.lang.String storeName)

      • sendHeartbeatToSystemStore

        public ControllerResponse sendHeartbeatToSystemStore​(java.lang.String storeName,
                                                     long heartbeatTimestamp)

      • configureActiveActiveReplicationForCluster

        public ControllerResponse configureActiveActiveReplicationForCluster​(boolean enableActiveActiveReplication,
                                                                     java.lang.String storeType,
                                                                     java.util.Optional<java.lang.String> regionsFilter)

      • checkResourceCleanupForStoreCreation

        public ControllerResponse checkResourceCleanupForStoreCreation​(java.lang.String storeName)

      • getQueryParamsToDiscoverCluster

        protected static QueryParams getQueryParamsToDiscoverCluster​(java.lang.String storeName)

      • getClusterStaleStores

        public ClusterStaleDataAuditResponse getClusterStaleStores​(java.lang.String clusterName,
                                                           java.lang.String parentControllerUrl)

      • getStoreLargestUsedVersion

        public VersionResponse getStoreLargestUsedVersion​(java.lang.String clusterName,
                                                  java.lang.String storeName)

      • getRegionPushDetails

        public RegionPushDetailsResponse getRegionPushDetails​(java.lang.String storeName,
                                                      boolean isPartitionDetailEnabled)

      • listStorePushInfo

        public StoreHealthAuditResponse listStorePushInfo​(java.lang.String storeName,
                                                  boolean isPartitionDetailEnabled)

      • discoverCluster

        public static D2ServiceDiscoveryResponse discoverCluster​(java.lang.String discoveryUrls,
                                                         java.lang.String storeName,
                                                         java.util.Optional<SSLFactory> sslFactory,
                                                         int retryAttempts)

      • prepareDataRecovery

        public ControllerResponse prepareDataRecovery​(java.lang.String sourceFabric,
                                              java.lang.String destinationFabric,
                                              java.lang.String storeName,
                                              int versionNumber,
                                              java.util.Optional<java.lang.Integer> sourceAmplificationFactor)

      • isStoreVersionReadyForDataRecovery

        public ReadyForDataRecoveryResponse isStoreVersionReadyForDataRecovery​(java.lang.String sourceFabric,
                                                                       java.lang.String destinationFabric,
                                                                       java.lang.String storeName,
                                                                       int versionNumber,
                                                                       java.util.Optional<java.lang.Integer> sourceAmplificationFactor)

      • dataRecovery

        public ControllerResponse dataRecovery​(java.lang.String sourceFabric,
                                       java.lang.String destinationFabric,
                                       java.lang.String storeName,
                                       int versionNumber,
                                       boolean sourceVersionIncluded,
                                       boolean copyAllVersionConfigs,
                                       java.util.Optional<Version> sourceVersion)

      • getAdminTopicMetadata

        public AdminTopicMetadataResponse getAdminTopicMetadata​(java.util.Optional<java.lang.String> storeName)

      • updateAdminTopicMetadata

        public ControllerResponse updateAdminTopicMetadata​(long executionId,
                                                   java.util.Optional<java.lang.String> storeName,
                                                   java.util.Optional<java.lang.Long> offset,
                                                   java.util.Optional<java.lang.Long> upstreamOffset)

      • deleteKafkaTopic

        public ControllerResponse deleteKafkaTopic​(java.lang.String topicName)

      • createStoragePersona

        public ControllerResponse createStoragePersona​(java.lang.String name,
                                               long quota,
                                               java.util.Set<java.lang.String> storesToEnforce,
                                               java.util.Set<java.lang.String> owners)

      • deleteStoragePersona

        public ControllerResponse deleteStoragePersona​(java.lang.String name)

      • getStoragePersonaAssociatedWithStore

        public StoragePersonaResponse getStoragePersonaAssociatedWithStore​(java.lang.String name)

      • removeStoreFromGraveyard

        public ControllerResponse removeStoreFromGraveyard​(java.lang.String storeName)

      • newParams

        protected QueryParams newParams()
        Add all global parameters in this method. Always use a form of this method to generate a new list of NameValuePair objects for making HTTP requests.
        Returns:

      • encodeQueryParams

        protected static java.lang.String encodeQueryParams​(QueryParams params)

      • getClusterName

        public java.lang.String getClusterName()

      • getLeaderControllerUrl

        public java.lang.String getLeaderControllerUrl()

      • getControllerDiscoveryUrls

        public java.util.Collection<java.lang.String> getControllerDiscoveryUrls()