Package com.linkedin.venice.utils

Class TestWriteUtils

  • public class TestWriteUtils
extends java.lang.Object

    • Field Detail

      • DEFAULT_USER_DATA_RECORD_COUNT

        public static final int DEFAULT_USER_DATA_RECORD_COUNT
        See Also:
        Constant Field Values

      • DEFAULT_USER_DATA_VALUE_PREFIX

        public static final java.lang.String DEFAULT_USER_DATA_VALUE_PREFIX
        See Also:
        Constant Field Values

      • STRING_SCHEMA

        public static final org.apache.avro.Schema STRING_SCHEMA

      • INT_SCHEMA

        public static final org.apache.avro.Schema INT_SCHEMA

      • USER_SCHEMA

        public static final org.apache.avro.Schema USER_SCHEMA

      • USER_WITH_DEFAULT_SCHEMA

        public static final org.apache.avro.Schema USER_WITH_DEFAULT_SCHEMA

      • SIMPLE_USER_WITH_DEFAULT_SCHEMA

        public static final org.apache.avro.Schema SIMPLE_USER_WITH_DEFAULT_SCHEMA

      • USER_WITH_FLOAT_ARRAY_SCHEMA

        public static final org.apache.avro.Schema USER_WITH_FLOAT_ARRAY_SCHEMA

      • USER_WITH_STRING_MAP_SCHEMA

        public static final org.apache.avro.Schema USER_WITH_STRING_MAP_SCHEMA

      • NAME_RECORD_V1_SCHEMA

        public static final org.apache.avro.Schema NAME_RECORD_V1_SCHEMA

      • NAME_RECORD_V2_SCHEMA

        public static final org.apache.avro.Schema NAME_RECORD_V2_SCHEMA

      • NAME_RECORD_V3_SCHEMA

        public static final org.apache.avro.Schema NAME_RECORD_V3_SCHEMA

      • NAME_RECORD_V4_SCHEMA

        public static final org.apache.avro.Schema NAME_RECORD_V4_SCHEMA

      • NAME_RECORD_V5_SCHEMA

        public static final org.apache.avro.Schema NAME_RECORD_V5_SCHEMA

      • NAME_RECORD_V6_SCHEMA

        public static final org.apache.avro.Schema NAME_RECORD_V6_SCHEMA

      • NAME_RECORD_V7_SCHEMA

        public static final org.apache.avro.Schema NAME_RECORD_V7_SCHEMA

      • NAME_RECORD_V8_SCHEMA

        public static final org.apache.avro.Schema NAME_RECORD_V8_SCHEMA

      • NAME_RECORD_V9_SCHEMA

        public static final org.apache.avro.Schema NAME_RECORD_V9_SCHEMA

      • NAME_RECORD_V10_SCHEMA

        public static final org.apache.avro.Schema NAME_RECORD_V10_SCHEMA

      • NAME_RECORD_V11_SCHEMA

        public static final org.apache.avro.Schema NAME_RECORD_V11_SCHEMA

      • UNION_RECORD_V1_SCHEMA

        public static final org.apache.avro.Schema UNION_RECORD_V1_SCHEMA

      • UNION_RECORD_V2_SCHEMA

        public static final org.apache.avro.Schema UNION_RECORD_V2_SCHEMA

      • UNION_RECORD_V3_SCHEMA

        public static final org.apache.avro.Schema UNION_RECORD_V3_SCHEMA

      • ETL_KEY_SCHEMA

        public static final org.apache.avro.Schema ETL_KEY_SCHEMA

      • ETL_VALUE_SCHEMA

        public static final org.apache.avro.Schema ETL_VALUE_SCHEMA

      • ETL_UNION_VALUE_WITH_NULL_SCHEMA

        public static final org.apache.avro.Schema ETL_UNION_VALUE_WITH_NULL_SCHEMA

      • ETL_UNION_VALUE_WITHOUT_NULL_SCHEMA

        public static final org.apache.avro.Schema ETL_UNION_VALUE_WITHOUT_NULL_SCHEMA

      • NAME_RECORD_V1_UPDATE_SCHEMA

        public static final org.apache.avro.Schema NAME_RECORD_V1_UPDATE_SCHEMA

      • INT_TO_STRING_SCHEMA

        public static final org.apache.avro.Schema INT_TO_STRING_SCHEMA

      • INT_TO_INT_SCHEMA

        public static final org.apache.avro.Schema INT_TO_INT_SCHEMA

      • STRING_TO_STRING_SCHEMA

        public static final org.apache.avro.Schema STRING_TO_STRING_SCHEMA

      • STRING_TO_NAME_RECORD_V1_SCHEMA

        public static final org.apache.avro.Schema STRING_TO_NAME_RECORD_V1_SCHEMA

      • STRING_TO_NAME_RECORD_V2_SCHEMA

        public static final org.apache.avro.Schema STRING_TO_NAME_RECORD_V2_SCHEMA

      • STRING_TO_NAME_RECORD_V3_SCHEMA

        public static final org.apache.avro.Schema STRING_TO_NAME_RECORD_V3_SCHEMA

      • STRING_TO_NAME_RECORD_V1_UPDATE_SCHEMA

        public static final org.apache.avro.Schema STRING_TO_NAME_RECORD_V1_UPDATE_SCHEMA

      • STRING_TO_STRING_WITH_EXTRA_FIELD_SCHEMA

        public static final org.apache.avro.Schema STRING_TO_STRING_WITH_EXTRA_FIELD_SCHEMA

      • STRING_TO_USER_WITH_STRING_MAP_SCHEMA

        public static final org.apache.avro.Schema STRING_TO_USER_WITH_STRING_MAP_SCHEMA

    • Constructor Detail

      • TestWriteUtils

        public TestWriteUtils()

    • Method Detail

      • getTempDataDirectory

        public static java.io.File getTempDataDirectory()

      • writeSimpleAvroFileWithStringToStringSchema

        public static org.apache.avro.Schema writeSimpleAvroFileWithStringToStringSchema​(java.io.File parentDir)
                                                                          throws java.io.IOException
        Throws:
        java.io.IOException

      • writeSimpleAvroFileWithStringToStringSchema

        public static org.apache.avro.Schema writeSimpleAvroFileWithStringToStringSchema​(java.io.File parentDir,
                                                                                 int recordCount)
                                                                          throws java.io.IOException
        Throws:
        java.io.IOException

      • writeSimpleAvroFileWithStringToStringSchema

        public static org.apache.avro.Schema writeSimpleAvroFileWithStringToStringSchema​(java.io.File parentDir,
                                                                                 int recordCount,
                                                                                 java.lang.String fileName)
                                                                          throws java.io.IOException
        Throws:
        java.io.IOException

      • writeSimpleAvroFileWithStringToStringWithExtraSchema

        public static org.apache.avro.Schema writeSimpleAvroFileWithStringToStringWithExtraSchema​(java.io.File parentDir)
                                                                                   throws java.io.IOException
        Throws:
        java.io.IOException

      • writeSimpleAvroFileWithStringToStringSchema

        public static org.apache.avro.Schema writeSimpleAvroFileWithStringToStringSchema​(java.io.File parentDir,
                                                                                 int recordCount,
                                                                                 int recordSizeMin)
                                                                          throws java.io.IOException
        Throws:
        java.io.IOException

      • writeSimpleAvroFileWithStringToV3Schema

        public static org.apache.avro.Schema writeSimpleAvroFileWithStringToV3Schema​(java.io.File parentDir,
                                                                             int recordCount,
                                                                             int recordSizeMin)
                                                                      throws java.io.IOException
        Throws:
        java.io.IOException

      • writeAlternateSimpleAvroFileWithStringToStringSchema

        public static org.apache.avro.Schema writeAlternateSimpleAvroFileWithStringToStringSchema​(java.io.File parentDir)
                                                                                   throws java.io.IOException
        Throws:
        java.io.IOException

      • writeSimpleAvroFileWithStringToStringSchema2

        public static org.apache.avro.Schema writeSimpleAvroFileWithStringToStringSchema2​(java.io.File parentDir)
                                                                           throws java.io.IOException
        This file overrides half of the value in writeSimpleAvroFileWithStringToStringSchema(File) and add some new values. It's designed to test incremental push
        Throws:
        java.io.IOException

      • writeSimpleAvroFileWithString2StringSchema3

        public static org.apache.avro.Schema writeSimpleAvroFileWithString2StringSchema3​(java.io.File parentDir)
                                                                          throws java.io.IOException
        This file add some new value in writeSimpleAvroFileWithStringToStringSchema(File) It's designed to test incremental push
        Throws:
        java.io.IOException

      • writeSimpleAvroFileWithDuplicateKey

        public static org.apache.avro.Schema writeSimpleAvroFileWithDuplicateKey​(java.io.File parentDir)
                                                                  throws java.io.IOException
        Throws:
        java.io.IOException

      • writeSimpleAvroFileWithCustomSize

        public static org.apache.avro.Schema writeSimpleAvroFileWithCustomSize​(java.io.File parentDir,
                                                                       int numberOfRecords,
                                                                       int minValueSize,
                                                                       int maxValueSize)
                                                                throws java.io.IOException
        Throws:
        java.io.IOException

      • writeSimpleAvroFileWithIntToStringSchema

        public static org.apache.avro.Schema writeSimpleAvroFileWithIntToStringSchema​(java.io.File parentDir)
                                                                       throws java.io.IOException
        Throws:
        java.io.IOException

      • writeSimpleAvroFileWithIntToStringSchema

        public static org.apache.avro.Schema writeSimpleAvroFileWithIntToStringSchema​(java.io.File parentDir,
                                                                              java.lang.String customValue,
                                                                              int numKeys)
                                                                       throws java.io.IOException
        Throws:
        java.io.IOException

      • writeSimpleAvroFileWithIntToIntSchema

        public static org.apache.avro.Schema writeSimpleAvroFileWithIntToIntSchema​(java.io.File parentDir,
                                                                           int numKeys)
                                                                    throws java.io.IOException
        Throws:
        java.io.IOException

      • writeInvalidAvroFile

        public static void writeInvalidAvroFile​(java.io.File parentDir,
                                        java.lang.String fileName)
                                 throws java.io.IOException
        Throws:
        java.io.IOException

      • writeEmptyAvroFile

        public static org.apache.avro.Schema writeEmptyAvroFile​(java.io.File parentDir,
                                                        org.apache.avro.Schema schema)
                                                 throws java.io.IOException
        Throws:
        java.io.IOException

      • writeEmptyAvroFile

        public static org.apache.avro.Schema writeEmptyAvroFile​(java.io.File parentDir,
                                                        java.lang.String fileName,
                                                        org.apache.avro.Schema schema)
                                                 throws java.io.IOException
        Throws:
        java.io.IOException

      • writeSimpleAvroFileWithStringToNameRecordV1Schema

        public static org.apache.avro.Schema writeSimpleAvroFileWithStringToNameRecordV1Schema​(java.io.File parentDir)
                                                                                throws java.io.IOException
        Throws:
        java.io.IOException

      • writeSimpleAvroFileWithStringToUserWithStringMapSchema

        public static org.apache.avro.Schema writeSimpleAvroFileWithStringToUserWithStringMapSchema​(java.io.File parentDir,
                                                                                            int itemsPerRecord)
                                                                                     throws java.io.IOException
        Throws:
        java.io.IOException

      • writeSimpleAvroFileWithStringToPartialUpdateOpRecordSchema

        public static org.apache.avro.Schema writeSimpleAvroFileWithStringToPartialUpdateOpRecordSchema​(java.io.File parentDir)
                                                                                         throws java.io.IOException
        Throws:
        java.io.IOException

      • writeSimpleAvroFileWithStringToPartialUpdateOpRecordSchema

        public static org.apache.avro.Schema writeSimpleAvroFileWithStringToPartialUpdateOpRecordSchema​(java.io.File parentDir,
                                                                                                int startIndex,
                                                                                                int endIndex)
                                                                                         throws java.io.IOException
        Throws:
        java.io.IOException

      • writeSimpleAvroFileWithASchemaWithAWrongDefaultValue

        public static org.apache.avro.Schema writeSimpleAvroFileWithASchemaWithAWrongDefaultValue​(java.io.File parentDir,
                                                                                          int numberOfRecords)
                                                                                   throws java.io.IOException
        Throws:
        java.io.IOException

      • writeAvroFileWithManyFloatsAndCustomTotalSize

        public static org.apache.avro.Schema writeAvroFileWithManyFloatsAndCustomTotalSize​(java.io.File parentDir,
                                                                                   int numberOfRecords,
                                                                                   int minValueSize,
                                                                                   int maxValueSize)
                                                                            throws java.io.IOException
        Throws:
        java.io.IOException

      • getRecordWithFloatArray

        public static org.apache.avro.generic.GenericRecord getRecordWithFloatArray​(org.apache.avro.Schema recordSchema,
                                                                            int index,
                                                                            int size)

      • getRecordWithStringMap

        public static org.apache.avro.generic.GenericRecord getRecordWithStringMap​(org.apache.avro.Schema recordSchema,
                                                                           int index,
                                                                           int count)

      • writeSimpleVsonFile

        public static KeyAndValueSchemas writeSimpleVsonFile​(java.io.File parentDir)
                                              throws java.io.IOException
        Throws:
        java.io.IOException

      • writeSchemaWithUnknownFieldIntoAvroFile

        public static org.apache.avro.Schema writeSchemaWithUnknownFieldIntoAvroFile​(java.io.File parentDir)
                                                                      throws java.io.IOException
        Throws:
        java.io.IOException

      • writeVsonByteAndShort

        public static KeyAndValueSchemas writeVsonByteAndShort​(java.io.File parentDir)
                                                throws java.io.IOException
        Throws:
        java.io.IOException

      • writeComplexVsonFile

        public static KeyAndValueSchemas writeComplexVsonFile​(java.io.File parentDir)
                                               throws java.io.IOException
        Throws:
        java.io.IOException

      • writeSimpleVsonFileWithUserSchema

        public static Pair<org.apache.avro.Schema,​org.apache.avro.Schema> writeSimpleVsonFileWithUserSchema​(java.io.File parentDir)
                                                                                                   throws java.io.IOException
        Throws:
        java.io.IOException

      • writeMultiLevelVsonFile

        public static KeyAndValueSchemas writeMultiLevelVsonFile​(java.io.File parentDir)
                                                  throws java.io.IOException
        Throws:
        java.io.IOException

      • writeMultiLevelVsonFile2

        public static Pair<VsonSchema,​VsonSchema> writeMultiLevelVsonFile2​(java.io.File parentDir)
                                                                  throws java.io.IOException
        Throws:
        java.io.IOException

      • defaultVPJPropsWithD2Routing

        public static java.util.Properties defaultVPJPropsWithD2Routing​(java.lang.String parentRegionName,
                                                                java.lang.String parentRegionD2ZkAddress,
                                                                java.util.Map<java.lang.String,​java.lang.String> childRegionNamesToZkAddress,
                                                                java.lang.String parentControllerD2ServiceName,
                                                                java.lang.String childControllerD2ServiceName,
                                                                java.lang.String inputDirPath,
                                                                java.lang.String storeName)

      • defaultVPJProps

        public static java.util.Properties defaultVPJProps​(java.lang.String veniceUrl,
                                                   java.lang.String inputDirPath,
                                                   java.lang.String storeName)

      • loadFileAsString

        public static java.lang.String loadFileAsString​(java.lang.String fileName)

      • loadSchemaFileFromResource

        public static java.lang.String loadSchemaFileFromResource​(java.lang.String fileName)

      • loadFileAsStringQuietlyWithErrorLogged

        public static java.lang.String loadFileAsStringQuietlyWithErrorLogged​(java.lang.String fileName)

      • writeSimpleAvroFileForValidateSchemaAndBuildDictMapperOutput

        public static org.apache.avro.Schema writeSimpleAvroFileForValidateSchemaAndBuildDictMapperOutput​(java.io.File parentDir,
                                                                                                  java.lang.String file,
                                                                                                  long inputFileDataSize,
                                                                                                  java.nio.ByteBuffer zstdDictionary,
                                                                                                  org.apache.avro.Schema avroSchema)
                                                                                           throws java.io.IOException
        Throws:
        java.io.IOException

      • writeETLFileWithUserSchema

        public static org.apache.avro.Schema writeETLFileWithUserSchema​(java.io.File parentDir)
                                                         throws java.io.IOException
        Throws:
        java.io.IOException

      • writeETLFileWithUserSchemaAndNullDefaultValue

        public static org.apache.avro.Schema writeETLFileWithUserSchemaAndNullDefaultValue​(java.io.File parentDir)
                                                                            throws java.io.IOException
        Throws:
        java.io.IOException

      • writeETLFileWithUnionWithNullSchema

        public static org.apache.avro.Schema writeETLFileWithUnionWithNullSchema​(java.io.File parentDir)
                                                                  throws java.io.IOException
        Throws:
        java.io.IOException

      • writeETLFileWithUnionWithoutNullSchema

        public static org.apache.avro.Schema writeETLFileWithUnionWithoutNullSchema​(java.io.File parentDir)
                                                                     throws java.io.IOException
        Throws:
        java.io.IOException

      • getETLFileSchema

        public static org.apache.avro.Schema getETLFileSchema​(org.apache.avro.Schema keySchema,
                                                      org.apache.avro.Schema valueSchema)

      • getETLFileSchemaWithNullDefaultValue

        public static org.apache.avro.Schema getETLFileSchemaWithNullDefaultValue​(org.apache.avro.Schema keySchema,
                                                                          org.apache.avro.Schema valueSchema)

      • runPushJob

        public static void runPushJob​(java.lang.String jobId,
                              java.util.Properties props)

      • runPushJob

        public static void runPushJob​(java.lang.String jobId,
                              java.util.Properties props,
                              java.util.function.Consumer<VenicePushJob> jobTransformer)