Package com.linkedin.venice.utils

Class TestWriteUtils

java.lang.Object
com.linkedin.venice.utils.TestWriteUtils
public class TestWriteUtils extends Object

  • Field Details

    • DEFAULT_USER_DATA_RECORD_COUNT

      public static final int DEFAULT_USER_DATA_RECORD_COUNT
      See Also:

    • DEFAULT_USER_DATA_VALUE_PREFIX

      public static final String DEFAULT_USER_DATA_VALUE_PREFIX
      See Also:

    • STRING_SCHEMA

      public static final org.apache.avro.Schema STRING_SCHEMA

    • INT_SCHEMA

      public static final org.apache.avro.Schema INT_SCHEMA

    • USER_SCHEMA

      public static final org.apache.avro.Schema USER_SCHEMA

    • USER_WITH_DEFAULT_SCHEMA

      public static final org.apache.avro.Schema USER_WITH_DEFAULT_SCHEMA

    • SINGLE_FIELD_RECORD_SCHEMA

      public static final org.apache.avro.Schema SINGLE_FIELD_RECORD_SCHEMA

    • SIMPLE_USER_WITH_DEFAULT_SCHEMA

      public static final org.apache.avro.Schema SIMPLE_USER_WITH_DEFAULT_SCHEMA

    • USER_WITH_FLOAT_ARRAY_SCHEMA

      public static final org.apache.avro.Schema USER_WITH_FLOAT_ARRAY_SCHEMA

    • USER_WITH_STRING_MAP_SCHEMA

      public static final org.apache.avro.Schema USER_WITH_STRING_MAP_SCHEMA

    • NAME_RECORD_V1_SCHEMA

      public static final org.apache.avro.Schema NAME_RECORD_V1_SCHEMA

    • NAME_RECORD_V2_SCHEMA

      public static final org.apache.avro.Schema NAME_RECORD_V2_SCHEMA

    • NAME_RECORD_V3_SCHEMA

      public static final org.apache.avro.Schema NAME_RECORD_V3_SCHEMA

    • NAME_RECORD_V4_SCHEMA

      public static final org.apache.avro.Schema NAME_RECORD_V4_SCHEMA

    • NAME_RECORD_V5_SCHEMA

      public static final org.apache.avro.Schema NAME_RECORD_V5_SCHEMA

    • NAME_RECORD_V6_SCHEMA

      public static final org.apache.avro.Schema NAME_RECORD_V6_SCHEMA

    • NAME_RECORD_V7_SCHEMA

      public static final org.apache.avro.Schema NAME_RECORD_V7_SCHEMA

    • NAME_RECORD_V8_SCHEMA

      public static final org.apache.avro.Schema NAME_RECORD_V8_SCHEMA

    • NAME_RECORD_V9_SCHEMA

      public static final org.apache.avro.Schema NAME_RECORD_V9_SCHEMA

    • NAME_RECORD_V10_SCHEMA

      public static final org.apache.avro.Schema NAME_RECORD_V10_SCHEMA

    • NAME_RECORD_V11_SCHEMA

      public static final org.apache.avro.Schema NAME_RECORD_V11_SCHEMA

    • UNION_RECORD_V1_SCHEMA

      public static final org.apache.avro.Schema UNION_RECORD_V1_SCHEMA

    • UNION_RECORD_V2_SCHEMA

      public static final org.apache.avro.Schema UNION_RECORD_V2_SCHEMA

    • UNION_RECORD_V3_SCHEMA

      public static final org.apache.avro.Schema UNION_RECORD_V3_SCHEMA

    • ETL_KEY_SCHEMA

      public static final org.apache.avro.Schema ETL_KEY_SCHEMA

    • ETL_VALUE_SCHEMA

      public static final org.apache.avro.Schema ETL_VALUE_SCHEMA

    • ETL_UNION_VALUE_WITH_NULL_SCHEMA

      public static final org.apache.avro.Schema ETL_UNION_VALUE_WITH_NULL_SCHEMA

    • ETL_UNION_VALUE_WITHOUT_NULL_SCHEMA

      public static final org.apache.avro.Schema ETL_UNION_VALUE_WITHOUT_NULL_SCHEMA

    • NAME_RECORD_V1_UPDATE_SCHEMA

      public static final org.apache.avro.Schema NAME_RECORD_V1_UPDATE_SCHEMA

    • INT_TO_STRING_SCHEMA

      public static final org.apache.avro.Schema INT_TO_STRING_SCHEMA

    • INT_TO_INT_SCHEMA

      public static final org.apache.avro.Schema INT_TO_INT_SCHEMA

    • STRING_TO_STRING_SCHEMA

      public static final org.apache.avro.Schema STRING_TO_STRING_SCHEMA

    • STRING_TO_NAME_RECORD_V1_SCHEMA

      public static final org.apache.avro.Schema STRING_TO_NAME_RECORD_V1_SCHEMA

    • STRING_TO_NAME_RECORD_V2_SCHEMA

      public static final org.apache.avro.Schema STRING_TO_NAME_RECORD_V2_SCHEMA

    • STRING_TO_NAME_RECORD_V3_SCHEMA

      public static final org.apache.avro.Schema STRING_TO_NAME_RECORD_V3_SCHEMA

    • STRING_TO_NAME_RECORD_V1_UPDATE_SCHEMA

      public static final org.apache.avro.Schema STRING_TO_NAME_RECORD_V1_UPDATE_SCHEMA

    • STRING_TO_STRING_WITH_EXTRA_FIELD_SCHEMA

      public static final org.apache.avro.Schema STRING_TO_STRING_WITH_EXTRA_FIELD_SCHEMA

    • STRING_TO_USER_WITH_STRING_MAP_SCHEMA

      public static final org.apache.avro.Schema STRING_TO_USER_WITH_STRING_MAP_SCHEMA

  • Constructor Details

    • TestWriteUtils

      public TestWriteUtils()

  • Method Details

    • getTempDataDirectory

      public static File getTempDataDirectory()

    • writeSimpleAvroFileWithStringToStringSchema

      public static org.apache.avro.Schema writeSimpleAvroFileWithStringToStringSchema(File parentDir) throws IOException
      Throws:
      IOException

    • writeSimpleAvroFileWithStringToStringSchema

      public static org.apache.avro.Schema writeSimpleAvroFileWithStringToStringSchema(File parentDir, int recordCount) throws IOException
      Throws:
      IOException

    • writeSimpleAvroFileWithStringToStringSchema

      public static org.apache.avro.Schema writeSimpleAvroFileWithStringToStringSchema(File parentDir, int recordCount, String fileName) throws IOException
      Throws:
      IOException

    • writeSimpleAvroFileWithStringToStringWithExtraSchema

      public static org.apache.avro.Schema writeSimpleAvroFileWithStringToStringWithExtraSchema(File parentDir) throws IOException
      Throws:
      IOException

    • writeSimpleAvroFileWithStringToStringSchema

      public static org.apache.avro.Schema writeSimpleAvroFileWithStringToStringSchema(File parentDir, int recordCount, int recordSizeMin) throws IOException
      Throws:
      IOException

    • writeSimpleAvroFileWithStringToV3Schema

      public static org.apache.avro.Schema writeSimpleAvroFileWithStringToV3Schema(File parentDir, int recordCount, int recordSizeMin) throws IOException
      Throws:
      IOException

    • writeAlternateSimpleAvroFileWithStringToStringSchema

      public static org.apache.avro.Schema writeAlternateSimpleAvroFileWithStringToStringSchema(File parentDir) throws IOException
      Throws:
      IOException

    • writeSimpleAvroFileWithStringToStringSchema2

      public static org.apache.avro.Schema writeSimpleAvroFileWithStringToStringSchema2(File parentDir) throws IOException
      This file overrides half of the value in writeSimpleAvroFileWithStringToStringSchema(File) and add some new values. It's designed to test incremental push
      Throws:
      IOException

    • writeSimpleAvroFileWithString2StringSchema3

      public static org.apache.avro.Schema writeSimpleAvroFileWithString2StringSchema3(File parentDir) throws IOException
      This file add some new value in writeSimpleAvroFileWithStringToStringSchema(File) It's designed to test incremental push
      Throws:
      IOException

    • writeSimpleAvroFileWithDuplicateKey

      public static org.apache.avro.Schema writeSimpleAvroFileWithDuplicateKey(File parentDir) throws IOException
      Throws:
      IOException

    • writeSimpleAvroFileWithCustomSize

      public static org.apache.avro.Schema writeSimpleAvroFileWithCustomSize(File parentDir, int numberOfRecords, int minValueSize, int maxValueSize) throws IOException
      Throws:
      IOException

    • writeSimpleAvroFileWithIntToStringSchema

      public static org.apache.avro.Schema writeSimpleAvroFileWithIntToStringSchema(File parentDir) throws IOException
      Throws:
      IOException

    • writeSimpleAvroFileWithIntToStringSchema

      public static org.apache.avro.Schema writeSimpleAvroFileWithIntToStringSchema(File parentDir, String customValue, int numKeys) throws IOException
      Throws:
      IOException

    • writeSimpleAvroFileWithIntToIntSchema

      public static org.apache.avro.Schema writeSimpleAvroFileWithIntToIntSchema(File parentDir, int numKeys) throws IOException
      Throws:
      IOException

    • writeInvalidAvroFile

      public static void writeInvalidAvroFile(File parentDir, String fileName) throws IOException
      Throws:
      IOException

    • writeEmptyAvroFile

      public static org.apache.avro.Schema writeEmptyAvroFile(File parentDir, org.apache.avro.Schema schema) throws IOException
      Throws:
      IOException

    • writeEmptyAvroFile

      public static org.apache.avro.Schema writeEmptyAvroFile(File parentDir, String fileName, org.apache.avro.Schema schema) throws IOException
      Throws:
      IOException

    • writeSimpleAvroFileWithStringToNameRecordV1Schema

      public static org.apache.avro.Schema writeSimpleAvroFileWithStringToNameRecordV1Schema(File parentDir) throws IOException
      Throws:
      IOException

    • writeSimpleAvroFile

      public static org.apache.avro.Schema writeSimpleAvroFile(File parentDir, org.apache.avro.Schema schema, Function<Integer,org.apache.avro.generic.GenericRecord> recordProvider) throws IOException
      Throws:
      IOException

    • writeSimpleAvroFileWithStringToUserWithStringMapSchema

      public static org.apache.avro.Schema writeSimpleAvroFileWithStringToUserWithStringMapSchema(File parentDir, int itemsPerRecord) throws IOException
      Throws:
      IOException

    • writeSimpleAvroFileWithStringToPartialUpdateOpRecordSchema

      public static org.apache.avro.Schema writeSimpleAvroFileWithStringToPartialUpdateOpRecordSchema(File parentDir) throws IOException
      Throws:
      IOException

    • writeSimpleAvroFileWithStringToPartialUpdateOpRecordSchema

      public static org.apache.avro.Schema writeSimpleAvroFileWithStringToPartialUpdateOpRecordSchema(File parentDir, int startIndex, int endIndex) throws IOException
      Throws:
      IOException

    • writeSimpleAvroFileWithASchemaWithAWrongDefaultValue

      public static org.apache.avro.Schema writeSimpleAvroFileWithASchemaWithAWrongDefaultValue(File parentDir, int numberOfRecords) throws IOException
      Throws:
      IOException

    • writeAvroFileWithManyFloatsAndCustomTotalSize

      public static org.apache.avro.Schema writeAvroFileWithManyFloatsAndCustomTotalSize(File parentDir, int numberOfRecords, int minValueSize, int maxValueSize) throws IOException
      Throws:
      IOException

    • getRecordWithFloatArray

      public static org.apache.avro.generic.GenericRecord getRecordWithFloatArray(org.apache.avro.Schema recordSchema, int index, int size)

    • getRecordWithStringMap

      public static org.apache.avro.generic.GenericRecord getRecordWithStringMap(org.apache.avro.Schema recordSchema, int index, int count)

    • writeSimpleVsonFile

      public static KeyAndValueSchemas writeSimpleVsonFile(File parentDir) throws IOException
      Throws:
      IOException

    • writeSchemaWithUnknownFieldIntoAvroFile

      public static org.apache.avro.Schema writeSchemaWithUnknownFieldIntoAvroFile(File parentDir) throws IOException
      Throws:
      IOException

    • writeVsonByteAndShort

      public static KeyAndValueSchemas writeVsonByteAndShort(File parentDir) throws IOException
      Throws:
      IOException

    • writeComplexVsonFile

      public static KeyAndValueSchemas writeComplexVsonFile(File parentDir) throws IOException
      Throws:
      IOException

    • writeSimpleVsonFileWithUserSchema

      public static Pair<org.apache.avro.Schema,org.apache.avro.Schema> writeSimpleVsonFileWithUserSchema(File parentDir) throws IOException
      Throws:
      IOException

    • writeMultiLevelVsonFile

      public static KeyAndValueSchemas writeMultiLevelVsonFile(File parentDir) throws IOException
      Throws:
      IOException

    • writeMultiLevelVsonFile2

      public static Pair<VsonSchema,VsonSchema> writeMultiLevelVsonFile2(File parentDir) throws IOException
      Throws:
      IOException

    • defaultVPJPropsWithD2Routing

      public static Properties defaultVPJPropsWithD2Routing(String parentRegionName, String parentRegionD2ZkAddress, Map<String,String> childRegionNamesToZkAddress, String parentControllerD2ServiceName, String childControllerD2ServiceName, String inputDirPath, String storeName)

    • defaultVPJProps

      public static Properties defaultVPJProps(String veniceUrl, String inputDirPath, String storeName)

    • loadFileAsString

      public static String loadFileAsString(String fileName)

    • loadSchemaFileFromResource

      public static String loadSchemaFileFromResource(String fileName)

    • loadFileAsStringQuietlyWithErrorLogged

      public static String loadFileAsStringQuietlyWithErrorLogged(String fileName)

    • updateStore

      public static void updateStore(String storeName, ControllerClient controllerClient, UpdateStoreQueryParams params)

    • writeSimpleAvroFileForValidateSchemaAndBuildDictMapperOutput

      public static org.apache.avro.Schema writeSimpleAvroFileForValidateSchemaAndBuildDictMapperOutput(File parentDir, String file, long inputFileDataSize, ByteBuffer zstdDictionary, org.apache.avro.Schema avroSchema) throws IOException
      Throws:
      IOException

    • writeETLFileWithUserSchema

      public static org.apache.avro.Schema writeETLFileWithUserSchema(File parentDir) throws IOException
      Throws:
      IOException

    • writeETLFileWithUserSchemaAndNullDefaultValue

      public static org.apache.avro.Schema writeETLFileWithUserSchemaAndNullDefaultValue(File parentDir) throws IOException
      Throws:
      IOException

    • writeETLFileWithUnionWithNullSchema

      public static org.apache.avro.Schema writeETLFileWithUnionWithNullSchema(File parentDir) throws IOException
      Throws:
      IOException

    • writeETLFileWithUnionWithoutNullSchema

      public static org.apache.avro.Schema writeETLFileWithUnionWithoutNullSchema(File parentDir) throws IOException
      Throws:
      IOException

    • getETLFileSchema

      public static org.apache.avro.Schema getETLFileSchema(org.apache.avro.Schema keySchema, org.apache.avro.Schema valueSchema)

    • getETLFileSchemaWithNullDefaultValue

      public static org.apache.avro.Schema getETLFileSchemaWithNullDefaultValue(org.apache.avro.Schema keySchema, org.apache.avro.Schema valueSchema)

    • runPushJob

      public static void runPushJob(String jobId, Properties props)

    • runPushJob

      public static void runPushJob(String jobId, Properties props, Consumer<VenicePushJob> jobTransformer)