Package com.linkedin.venice.utils
Class TestWriteUtils
- java.lang.Object
-
- com.linkedin.venice.utils.TestWriteUtils
-
public class TestWriteUtils extends java.lang.Object
-
-
Nested Class Summary
Nested Classes Modifier and Type Class Description static class
TestWriteUtils.TestRecordType
static class
TestWriteUtils.TestTargetedField
-
Field Summary
Fields Modifier and Type Field Description static int
DEFAULT_USER_DATA_RECORD_COUNT
static java.lang.String
DEFAULT_USER_DATA_VALUE_PREFIX
static org.apache.avro.Schema
ETL_KEY_SCHEMA
static org.apache.avro.Schema
ETL_UNION_VALUE_WITH_NULL_SCHEMA
static org.apache.avro.Schema
ETL_UNION_VALUE_WITHOUT_NULL_SCHEMA
static org.apache.avro.Schema
ETL_VALUE_SCHEMA
static org.apache.avro.Schema
INT_SCHEMA
static org.apache.avro.Schema
INT_TO_STRING_SCHEMA
static org.apache.avro.Schema
NAME_RECORD_V1_SCHEMA
static org.apache.avro.Schema
NAME_RECORD_V1_UPDATE_SCHEMA
static org.apache.avro.Schema
NAME_RECORD_V2_SCHEMA
static org.apache.avro.Schema
NAME_RECORD_V3_SCHEMA
static org.apache.avro.Schema
NAME_RECORD_V4_SCHEMA
static org.apache.avro.Schema
NAME_RECORD_V5_SCHEMA
static org.apache.avro.Schema
NAME_RECORD_V6_SCHEMA
static org.apache.avro.Schema
SIMPLE_USER_WITH_DEFAULT_SCHEMA
static org.apache.avro.Schema
STRING_SCHEMA
static org.apache.avro.Schema
STRING_TO_NAME_RECORD_V1_SCHEMA
static org.apache.avro.Schema
STRING_TO_NAME_RECORD_V1_UPDATE_SCHEMA
static org.apache.avro.Schema
STRING_TO_NAME_RECORD_V2_SCHEMA
static org.apache.avro.Schema
STRING_TO_NAME_RECORD_V3_SCHEMA
static org.apache.avro.Schema
STRING_TO_STRING_SCHEMA
static org.apache.avro.Schema
STRING_TO_STRING_WITH_EXTRA_FIELD_SCHEMA
static org.apache.avro.Schema
STRING_TO_USER_WITH_STRING_MAP_SCHEMA
static org.apache.avro.Schema
UNION_RECORD_V1_SCHEMA
static org.apache.avro.Schema
UNION_RECORD_V2_SCHEMA
static org.apache.avro.Schema
UNION_RECORD_V3_SCHEMA
static org.apache.avro.Schema
USER_SCHEMA
static org.apache.avro.Schema
USER_WITH_DEFAULT_SCHEMA
static org.apache.avro.Schema
USER_WITH_FLOAT_ARRAY_SCHEMA
static org.apache.avro.Schema
USER_WITH_STRING_MAP_SCHEMA
-
Constructor Summary
Constructors Constructor Description TestWriteUtils()
-
Method Summary
All Methods Static Methods Concrete Methods Modifier and Type Method Description static java.util.Properties
defaultVPJProps(java.lang.String veniceUrl, java.lang.String inputDirPath, java.lang.String storeName)
static java.util.Properties
defaultVPJPropsWithD2Routing(java.lang.String parentRegionName, java.lang.String parentRegionD2ZkAddress, java.util.Map<java.lang.String,java.lang.String> childRegionNamesToZkAddress, java.lang.String parentControllerD2ServiceName, java.lang.String childControllerD2ServiceName, java.lang.String inputDirPath, java.lang.String storeName)
static org.apache.avro.Schema
getETLFileSchema(org.apache.avro.Schema keySchema, org.apache.avro.Schema valueSchema)
static org.apache.avro.Schema
getETLFileSchemaWithNullDefaultValue(org.apache.avro.Schema keySchema, org.apache.avro.Schema valueSchema)
static org.apache.avro.generic.GenericRecord
getRecordWithFloatArray(org.apache.avro.Schema recordSchema, int index, int size)
static org.apache.avro.generic.GenericRecord
getRecordWithStringMap(org.apache.avro.Schema recordSchema, int index, int count)
static java.io.File
getTempDataDirectory()
static java.lang.String
loadFileAsString(java.lang.String fileName)
static java.lang.String
loadFileAsStringQuietlyWithErrorLogged(java.lang.String fileName)
static java.lang.String
loadSchemaFileFromResource(java.lang.String fileName)
static void
runPushJob(java.lang.String jobId, java.util.Properties props)
static void
runPushJob(java.lang.String jobId, java.util.Properties props, java.util.function.Consumer<VenicePushJob> jobTransformer)
static void
updateStore(java.lang.String storeName, ControllerClient controllerClient, UpdateStoreQueryParams params)
static org.apache.avro.Schema
writeAlternateSimpleAvroFileWithStringToStringSchema(java.io.File parentDir)
static org.apache.avro.Schema
writeAvroFileWithManyFloatsAndCustomTotalSize(java.io.File parentDir, int numberOfRecords, int minValueSize, int maxValueSize)
static KeyAndValueSchemas
writeComplexVsonFile(java.io.File parentDir)
static org.apache.avro.Schema
writeEmptyAvroFile(java.io.File parentDir, java.lang.String fileName, org.apache.avro.Schema schema)
static org.apache.avro.Schema
writeEmptyAvroFile(java.io.File parentDir, org.apache.avro.Schema schema)
static org.apache.avro.Schema
writeETLFileWithUnionWithNullSchema(java.io.File parentDir)
static org.apache.avro.Schema
writeETLFileWithUnionWithoutNullSchema(java.io.File parentDir)
static org.apache.avro.Schema
writeETLFileWithUserSchema(java.io.File parentDir)
static org.apache.avro.Schema
writeETLFileWithUserSchemaAndNullDefaultValue(java.io.File parentDir)
static void
writeInvalidAvroFile(java.io.File parentDir, java.lang.String fileName)
static KeyAndValueSchemas
writeMultiLevelVsonFile(java.io.File parentDir)
static Pair<VsonSchema,VsonSchema>
writeMultiLevelVsonFile2(java.io.File parentDir)
static org.apache.avro.Schema
writeSchemaWithUnknownFieldIntoAvroFile(java.io.File parentDir)
static org.apache.avro.Schema
writeSimpleAvroFileForValidateSchemaAndBuildDictMapperOutput(java.io.File parentDir, java.lang.String file, long inputFileDataSize, java.nio.ByteBuffer zstdDictionary, org.apache.avro.Schema avroSchema)
static org.apache.avro.Schema
writeSimpleAvroFileWithASchemaWithAWrongDefaultValue(java.io.File parentDir, int numberOfRecords)
static org.apache.avro.Schema
writeSimpleAvroFileWithCustomSize(java.io.File parentDir, int numberOfRecords, int minValueSize, int maxValueSize)
static org.apache.avro.Schema
writeSimpleAvroFileWithDuplicateKey(java.io.File parentDir)
static org.apache.avro.Schema
writeSimpleAvroFileWithIntToStringSchema(java.io.File parentDir)
static org.apache.avro.Schema
writeSimpleAvroFileWithString2StringSchema3(java.io.File parentDir)
This file add some new value inwriteSimpleAvroFileWithStringToStringSchema(File)
It's designed to test incremental pushstatic org.apache.avro.Schema
writeSimpleAvroFileWithStringToNameRecordV1Schema(java.io.File parentDir)
static org.apache.avro.Schema
writeSimpleAvroFileWithStringToPartialUpdateOpRecordSchema(java.io.File parentDir)
static org.apache.avro.Schema
writeSimpleAvroFileWithStringToPartialUpdateOpRecordSchema(java.io.File parentDir, int startIndex, int endIndex)
static org.apache.avro.Schema
writeSimpleAvroFileWithStringToStringSchema(java.io.File parentDir)
static org.apache.avro.Schema
writeSimpleAvroFileWithStringToStringSchema(java.io.File parentDir, int recordCount)
static org.apache.avro.Schema
writeSimpleAvroFileWithStringToStringSchema(java.io.File parentDir, int recordCount, int recordSizeMin)
static org.apache.avro.Schema
writeSimpleAvroFileWithStringToStringSchema(java.io.File parentDir, int recordCount, java.lang.String fileName)
static org.apache.avro.Schema
writeSimpleAvroFileWithStringToStringSchema2(java.io.File parentDir)
This file overrides half of the value inwriteSimpleAvroFileWithStringToStringSchema(File)
and add some new values.static org.apache.avro.Schema
writeSimpleAvroFileWithStringToStringWithExtraSchema(java.io.File parentDir)
static org.apache.avro.Schema
writeSimpleAvroFileWithStringToUserWithStringMapSchema(java.io.File parentDir, int itemsPerRecord)
static org.apache.avro.Schema
writeSimpleAvroFileWithStringToV3Schema(java.io.File parentDir, int recordCount, int recordSizeMin)
static KeyAndValueSchemas
writeSimpleVsonFile(java.io.File parentDir)
static Pair<org.apache.avro.Schema,org.apache.avro.Schema>
writeSimpleVsonFileWithUserSchema(java.io.File parentDir)
static KeyAndValueSchemas
writeVsonByteAndShort(java.io.File parentDir)
-
-
-
Field Detail
-
DEFAULT_USER_DATA_RECORD_COUNT
public static final int DEFAULT_USER_DATA_RECORD_COUNT
- See Also:
- Constant Field Values
-
DEFAULT_USER_DATA_VALUE_PREFIX
public static final java.lang.String DEFAULT_USER_DATA_VALUE_PREFIX
- See Also:
- Constant Field Values
-
STRING_SCHEMA
public static final org.apache.avro.Schema STRING_SCHEMA
-
INT_SCHEMA
public static final org.apache.avro.Schema INT_SCHEMA
-
USER_SCHEMA
public static final org.apache.avro.Schema USER_SCHEMA
-
USER_WITH_DEFAULT_SCHEMA
public static final org.apache.avro.Schema USER_WITH_DEFAULT_SCHEMA
-
SIMPLE_USER_WITH_DEFAULT_SCHEMA
public static final org.apache.avro.Schema SIMPLE_USER_WITH_DEFAULT_SCHEMA
-
USER_WITH_FLOAT_ARRAY_SCHEMA
public static final org.apache.avro.Schema USER_WITH_FLOAT_ARRAY_SCHEMA
-
USER_WITH_STRING_MAP_SCHEMA
public static final org.apache.avro.Schema USER_WITH_STRING_MAP_SCHEMA
-
NAME_RECORD_V1_SCHEMA
public static final org.apache.avro.Schema NAME_RECORD_V1_SCHEMA
-
NAME_RECORD_V2_SCHEMA
public static final org.apache.avro.Schema NAME_RECORD_V2_SCHEMA
-
NAME_RECORD_V3_SCHEMA
public static final org.apache.avro.Schema NAME_RECORD_V3_SCHEMA
-
NAME_RECORD_V4_SCHEMA
public static final org.apache.avro.Schema NAME_RECORD_V4_SCHEMA
-
NAME_RECORD_V5_SCHEMA
public static final org.apache.avro.Schema NAME_RECORD_V5_SCHEMA
-
NAME_RECORD_V6_SCHEMA
public static final org.apache.avro.Schema NAME_RECORD_V6_SCHEMA
-
UNION_RECORD_V1_SCHEMA
public static final org.apache.avro.Schema UNION_RECORD_V1_SCHEMA
-
UNION_RECORD_V2_SCHEMA
public static final org.apache.avro.Schema UNION_RECORD_V2_SCHEMA
-
UNION_RECORD_V3_SCHEMA
public static final org.apache.avro.Schema UNION_RECORD_V3_SCHEMA
-
ETL_KEY_SCHEMA
public static final org.apache.avro.Schema ETL_KEY_SCHEMA
-
ETL_VALUE_SCHEMA
public static final org.apache.avro.Schema ETL_VALUE_SCHEMA
-
ETL_UNION_VALUE_WITH_NULL_SCHEMA
public static final org.apache.avro.Schema ETL_UNION_VALUE_WITH_NULL_SCHEMA
-
ETL_UNION_VALUE_WITHOUT_NULL_SCHEMA
public static final org.apache.avro.Schema ETL_UNION_VALUE_WITHOUT_NULL_SCHEMA
-
NAME_RECORD_V1_UPDATE_SCHEMA
public static final org.apache.avro.Schema NAME_RECORD_V1_UPDATE_SCHEMA
-
INT_TO_STRING_SCHEMA
public static final org.apache.avro.Schema INT_TO_STRING_SCHEMA
-
STRING_TO_STRING_SCHEMA
public static final org.apache.avro.Schema STRING_TO_STRING_SCHEMA
-
STRING_TO_NAME_RECORD_V1_SCHEMA
public static final org.apache.avro.Schema STRING_TO_NAME_RECORD_V1_SCHEMA
-
STRING_TO_NAME_RECORD_V2_SCHEMA
public static final org.apache.avro.Schema STRING_TO_NAME_RECORD_V2_SCHEMA
-
STRING_TO_NAME_RECORD_V3_SCHEMA
public static final org.apache.avro.Schema STRING_TO_NAME_RECORD_V3_SCHEMA
-
STRING_TO_NAME_RECORD_V1_UPDATE_SCHEMA
public static final org.apache.avro.Schema STRING_TO_NAME_RECORD_V1_UPDATE_SCHEMA
-
STRING_TO_STRING_WITH_EXTRA_FIELD_SCHEMA
public static final org.apache.avro.Schema STRING_TO_STRING_WITH_EXTRA_FIELD_SCHEMA
-
STRING_TO_USER_WITH_STRING_MAP_SCHEMA
public static final org.apache.avro.Schema STRING_TO_USER_WITH_STRING_MAP_SCHEMA
-
-
Method Detail
-
getTempDataDirectory
public static java.io.File getTempDataDirectory()
-
writeSimpleAvroFileWithStringToStringSchema
public static org.apache.avro.Schema writeSimpleAvroFileWithStringToStringSchema(java.io.File parentDir) throws java.io.IOException
- Throws:
java.io.IOException
-
writeSimpleAvroFileWithStringToStringSchema
public static org.apache.avro.Schema writeSimpleAvroFileWithStringToStringSchema(java.io.File parentDir, int recordCount) throws java.io.IOException
- Throws:
java.io.IOException
-
writeSimpleAvroFileWithStringToStringSchema
public static org.apache.avro.Schema writeSimpleAvroFileWithStringToStringSchema(java.io.File parentDir, int recordCount, java.lang.String fileName) throws java.io.IOException
- Throws:
java.io.IOException
-
writeSimpleAvroFileWithStringToStringWithExtraSchema
public static org.apache.avro.Schema writeSimpleAvroFileWithStringToStringWithExtraSchema(java.io.File parentDir) throws java.io.IOException
- Throws:
java.io.IOException
-
writeSimpleAvroFileWithStringToStringSchema
public static org.apache.avro.Schema writeSimpleAvroFileWithStringToStringSchema(java.io.File parentDir, int recordCount, int recordSizeMin) throws java.io.IOException
- Throws:
java.io.IOException
-
writeSimpleAvroFileWithStringToV3Schema
public static org.apache.avro.Schema writeSimpleAvroFileWithStringToV3Schema(java.io.File parentDir, int recordCount, int recordSizeMin) throws java.io.IOException
- Throws:
java.io.IOException
-
writeAlternateSimpleAvroFileWithStringToStringSchema
public static org.apache.avro.Schema writeAlternateSimpleAvroFileWithStringToStringSchema(java.io.File parentDir) throws java.io.IOException
- Throws:
java.io.IOException
-
writeSimpleAvroFileWithStringToStringSchema2
public static org.apache.avro.Schema writeSimpleAvroFileWithStringToStringSchema2(java.io.File parentDir) throws java.io.IOException
This file overrides half of the value inwriteSimpleAvroFileWithStringToStringSchema(File)
and add some new values. It's designed to test incremental push- Throws:
java.io.IOException
-
writeSimpleAvroFileWithString2StringSchema3
public static org.apache.avro.Schema writeSimpleAvroFileWithString2StringSchema3(java.io.File parentDir) throws java.io.IOException
This file add some new value inwriteSimpleAvroFileWithStringToStringSchema(File)
It's designed to test incremental push- Throws:
java.io.IOException
-
writeSimpleAvroFileWithDuplicateKey
public static org.apache.avro.Schema writeSimpleAvroFileWithDuplicateKey(java.io.File parentDir) throws java.io.IOException
- Throws:
java.io.IOException
-
writeSimpleAvroFileWithCustomSize
public static org.apache.avro.Schema writeSimpleAvroFileWithCustomSize(java.io.File parentDir, int numberOfRecords, int minValueSize, int maxValueSize) throws java.io.IOException
- Throws:
java.io.IOException
-
writeSimpleAvroFileWithIntToStringSchema
public static org.apache.avro.Schema writeSimpleAvroFileWithIntToStringSchema(java.io.File parentDir) throws java.io.IOException
- Throws:
java.io.IOException
-
writeInvalidAvroFile
public static void writeInvalidAvroFile(java.io.File parentDir, java.lang.String fileName) throws java.io.IOException
- Throws:
java.io.IOException
-
writeEmptyAvroFile
public static org.apache.avro.Schema writeEmptyAvroFile(java.io.File parentDir, org.apache.avro.Schema schema) throws java.io.IOException
- Throws:
java.io.IOException
-
writeEmptyAvroFile
public static org.apache.avro.Schema writeEmptyAvroFile(java.io.File parentDir, java.lang.String fileName, org.apache.avro.Schema schema) throws java.io.IOException
- Throws:
java.io.IOException
-
writeSimpleAvroFileWithStringToNameRecordV1Schema
public static org.apache.avro.Schema writeSimpleAvroFileWithStringToNameRecordV1Schema(java.io.File parentDir) throws java.io.IOException
- Throws:
java.io.IOException
-
writeSimpleAvroFileWithStringToUserWithStringMapSchema
public static org.apache.avro.Schema writeSimpleAvroFileWithStringToUserWithStringMapSchema(java.io.File parentDir, int itemsPerRecord) throws java.io.IOException
- Throws:
java.io.IOException
-
writeSimpleAvroFileWithStringToPartialUpdateOpRecordSchema
public static org.apache.avro.Schema writeSimpleAvroFileWithStringToPartialUpdateOpRecordSchema(java.io.File parentDir) throws java.io.IOException
- Throws:
java.io.IOException
-
writeSimpleAvroFileWithStringToPartialUpdateOpRecordSchema
public static org.apache.avro.Schema writeSimpleAvroFileWithStringToPartialUpdateOpRecordSchema(java.io.File parentDir, int startIndex, int endIndex) throws java.io.IOException
- Throws:
java.io.IOException
-
writeSimpleAvroFileWithASchemaWithAWrongDefaultValue
public static org.apache.avro.Schema writeSimpleAvroFileWithASchemaWithAWrongDefaultValue(java.io.File parentDir, int numberOfRecords) throws java.io.IOException
- Throws:
java.io.IOException
-
writeAvroFileWithManyFloatsAndCustomTotalSize
public static org.apache.avro.Schema writeAvroFileWithManyFloatsAndCustomTotalSize(java.io.File parentDir, int numberOfRecords, int minValueSize, int maxValueSize) throws java.io.IOException
- Throws:
java.io.IOException
-
getRecordWithFloatArray
public static org.apache.avro.generic.GenericRecord getRecordWithFloatArray(org.apache.avro.Schema recordSchema, int index, int size)
-
getRecordWithStringMap
public static org.apache.avro.generic.GenericRecord getRecordWithStringMap(org.apache.avro.Schema recordSchema, int index, int count)
-
writeSimpleVsonFile
public static KeyAndValueSchemas writeSimpleVsonFile(java.io.File parentDir) throws java.io.IOException
- Throws:
java.io.IOException
-
writeSchemaWithUnknownFieldIntoAvroFile
public static org.apache.avro.Schema writeSchemaWithUnknownFieldIntoAvroFile(java.io.File parentDir) throws java.io.IOException
- Throws:
java.io.IOException
-
writeVsonByteAndShort
public static KeyAndValueSchemas writeVsonByteAndShort(java.io.File parentDir) throws java.io.IOException
- Throws:
java.io.IOException
-
writeComplexVsonFile
public static KeyAndValueSchemas writeComplexVsonFile(java.io.File parentDir) throws java.io.IOException
- Throws:
java.io.IOException
-
writeSimpleVsonFileWithUserSchema
public static Pair<org.apache.avro.Schema,org.apache.avro.Schema> writeSimpleVsonFileWithUserSchema(java.io.File parentDir) throws java.io.IOException
- Throws:
java.io.IOException
-
writeMultiLevelVsonFile
public static KeyAndValueSchemas writeMultiLevelVsonFile(java.io.File parentDir) throws java.io.IOException
- Throws:
java.io.IOException
-
writeMultiLevelVsonFile2
public static Pair<VsonSchema,VsonSchema> writeMultiLevelVsonFile2(java.io.File parentDir) throws java.io.IOException
- Throws:
java.io.IOException
-
defaultVPJPropsWithD2Routing
public static java.util.Properties defaultVPJPropsWithD2Routing(java.lang.String parentRegionName, java.lang.String parentRegionD2ZkAddress, java.util.Map<java.lang.String,java.lang.String> childRegionNamesToZkAddress, java.lang.String parentControllerD2ServiceName, java.lang.String childControllerD2ServiceName, java.lang.String inputDirPath, java.lang.String storeName)
-
defaultVPJProps
public static java.util.Properties defaultVPJProps(java.lang.String veniceUrl, java.lang.String inputDirPath, java.lang.String storeName)
-
loadFileAsString
public static java.lang.String loadFileAsString(java.lang.String fileName)
-
loadSchemaFileFromResource
public static java.lang.String loadSchemaFileFromResource(java.lang.String fileName)
-
loadFileAsStringQuietlyWithErrorLogged
public static java.lang.String loadFileAsStringQuietlyWithErrorLogged(java.lang.String fileName)
-
updateStore
public static void updateStore(java.lang.String storeName, ControllerClient controllerClient, UpdateStoreQueryParams params)
-
writeSimpleAvroFileForValidateSchemaAndBuildDictMapperOutput
public static org.apache.avro.Schema writeSimpleAvroFileForValidateSchemaAndBuildDictMapperOutput(java.io.File parentDir, java.lang.String file, long inputFileDataSize, java.nio.ByteBuffer zstdDictionary, org.apache.avro.Schema avroSchema) throws java.io.IOException
- Throws:
java.io.IOException
-
writeETLFileWithUserSchema
public static org.apache.avro.Schema writeETLFileWithUserSchema(java.io.File parentDir) throws java.io.IOException
- Throws:
java.io.IOException
-
writeETLFileWithUserSchemaAndNullDefaultValue
public static org.apache.avro.Schema writeETLFileWithUserSchemaAndNullDefaultValue(java.io.File parentDir) throws java.io.IOException
- Throws:
java.io.IOException
-
writeETLFileWithUnionWithNullSchema
public static org.apache.avro.Schema writeETLFileWithUnionWithNullSchema(java.io.File parentDir) throws java.io.IOException
- Throws:
java.io.IOException
-
writeETLFileWithUnionWithoutNullSchema
public static org.apache.avro.Schema writeETLFileWithUnionWithoutNullSchema(java.io.File parentDir) throws java.io.IOException
- Throws:
java.io.IOException
-
getETLFileSchema
public static org.apache.avro.Schema getETLFileSchema(org.apache.avro.Schema keySchema, org.apache.avro.Schema valueSchema)
-
getETLFileSchemaWithNullDefaultValue
public static org.apache.avro.Schema getETLFileSchemaWithNullDefaultValue(org.apache.avro.Schema keySchema, org.apache.avro.Schema valueSchema)
-
runPushJob
public static void runPushJob(java.lang.String jobId, java.util.Properties props)
-
runPushJob
public static void runPushJob(java.lang.String jobId, java.util.Properties props, java.util.function.Consumer<VenicePushJob> jobTransformer)
-
-