Class TestWriteUtils


  • public class TestWriteUtils
    extends java.lang.Object
    • Field Detail

      • DEFAULT_USER_DATA_RECORD_COUNT

        public static final int DEFAULT_USER_DATA_RECORD_COUNT
        See Also:
        Constant Field Values
      • DEFAULT_USER_DATA_VALUE_PREFIX

        public static final java.lang.String DEFAULT_USER_DATA_VALUE_PREFIX
        See Also:
        Constant Field Values
      • STRING_SCHEMA

        public static final org.apache.avro.Schema STRING_SCHEMA
      • INT_SCHEMA

        public static final org.apache.avro.Schema INT_SCHEMA
      • USER_SCHEMA

        public static final org.apache.avro.Schema USER_SCHEMA
      • USER_WITH_DEFAULT_SCHEMA

        public static final org.apache.avro.Schema USER_WITH_DEFAULT_SCHEMA
      • SIMPLE_USER_WITH_DEFAULT_SCHEMA

        public static final org.apache.avro.Schema SIMPLE_USER_WITH_DEFAULT_SCHEMA
      • USER_WITH_FLOAT_ARRAY_SCHEMA

        public static final org.apache.avro.Schema USER_WITH_FLOAT_ARRAY_SCHEMA
      • USER_WITH_STRING_MAP_SCHEMA

        public static final org.apache.avro.Schema USER_WITH_STRING_MAP_SCHEMA
      • NAME_RECORD_V1_SCHEMA

        public static final org.apache.avro.Schema NAME_RECORD_V1_SCHEMA
      • NAME_RECORD_V2_SCHEMA

        public static final org.apache.avro.Schema NAME_RECORD_V2_SCHEMA
      • NAME_RECORD_V3_SCHEMA

        public static final org.apache.avro.Schema NAME_RECORD_V3_SCHEMA
      • NAME_RECORD_V4_SCHEMA

        public static final org.apache.avro.Schema NAME_RECORD_V4_SCHEMA
      • NAME_RECORD_V5_SCHEMA

        public static final org.apache.avro.Schema NAME_RECORD_V5_SCHEMA
      • NAME_RECORD_V6_SCHEMA

        public static final org.apache.avro.Schema NAME_RECORD_V6_SCHEMA
      • UNION_RECORD_V1_SCHEMA

        public static final org.apache.avro.Schema UNION_RECORD_V1_SCHEMA
      • UNION_RECORD_V2_SCHEMA

        public static final org.apache.avro.Schema UNION_RECORD_V2_SCHEMA
      • UNION_RECORD_V3_SCHEMA

        public static final org.apache.avro.Schema UNION_RECORD_V3_SCHEMA
      • ETL_KEY_SCHEMA

        public static final org.apache.avro.Schema ETL_KEY_SCHEMA
      • ETL_VALUE_SCHEMA

        public static final org.apache.avro.Schema ETL_VALUE_SCHEMA
      • ETL_UNION_VALUE_WITH_NULL_SCHEMA

        public static final org.apache.avro.Schema ETL_UNION_VALUE_WITH_NULL_SCHEMA
      • ETL_UNION_VALUE_WITHOUT_NULL_SCHEMA

        public static final org.apache.avro.Schema ETL_UNION_VALUE_WITHOUT_NULL_SCHEMA
      • NAME_RECORD_V1_UPDATE_SCHEMA

        public static final org.apache.avro.Schema NAME_RECORD_V1_UPDATE_SCHEMA
      • INT_TO_STRING_SCHEMA

        public static final org.apache.avro.Schema INT_TO_STRING_SCHEMA
      • STRING_TO_STRING_SCHEMA

        public static final org.apache.avro.Schema STRING_TO_STRING_SCHEMA
      • STRING_TO_NAME_RECORD_V1_SCHEMA

        public static final org.apache.avro.Schema STRING_TO_NAME_RECORD_V1_SCHEMA
      • STRING_TO_NAME_RECORD_V2_SCHEMA

        public static final org.apache.avro.Schema STRING_TO_NAME_RECORD_V2_SCHEMA
      • STRING_TO_NAME_RECORD_V3_SCHEMA

        public static final org.apache.avro.Schema STRING_TO_NAME_RECORD_V3_SCHEMA
      • STRING_TO_NAME_RECORD_V1_UPDATE_SCHEMA

        public static final org.apache.avro.Schema STRING_TO_NAME_RECORD_V1_UPDATE_SCHEMA
      • STRING_TO_STRING_WITH_EXTRA_FIELD_SCHEMA

        public static final org.apache.avro.Schema STRING_TO_STRING_WITH_EXTRA_FIELD_SCHEMA
      • STRING_TO_USER_WITH_STRING_MAP_SCHEMA

        public static final org.apache.avro.Schema STRING_TO_USER_WITH_STRING_MAP_SCHEMA
    • Constructor Detail

      • TestWriteUtils

        public TestWriteUtils()
    • Method Detail

      • getTempDataDirectory

        public static java.io.File getTempDataDirectory()
      • writeSimpleAvroFileWithStringToStringSchema

        public static org.apache.avro.Schema writeSimpleAvroFileWithStringToStringSchema​(java.io.File parentDir)
                                                                                  throws java.io.IOException
        Throws:
        java.io.IOException
      • writeSimpleAvroFileWithStringToStringSchema

        public static org.apache.avro.Schema writeSimpleAvroFileWithStringToStringSchema​(java.io.File parentDir,
                                                                                         int recordCount)
                                                                                  throws java.io.IOException
        Throws:
        java.io.IOException
      • writeSimpleAvroFileWithStringToStringSchema

        public static org.apache.avro.Schema writeSimpleAvroFileWithStringToStringSchema​(java.io.File parentDir,
                                                                                         int recordCount,
                                                                                         java.lang.String fileName)
                                                                                  throws java.io.IOException
        Throws:
        java.io.IOException
      • writeSimpleAvroFileWithStringToStringWithExtraSchema

        public static org.apache.avro.Schema writeSimpleAvroFileWithStringToStringWithExtraSchema​(java.io.File parentDir)
                                                                                           throws java.io.IOException
        Throws:
        java.io.IOException
      • writeSimpleAvroFileWithStringToStringSchema

        public static org.apache.avro.Schema writeSimpleAvroFileWithStringToStringSchema​(java.io.File parentDir,
                                                                                         int recordCount,
                                                                                         int recordSizeMin)
                                                                                  throws java.io.IOException
        Throws:
        java.io.IOException
      • writeSimpleAvroFileWithStringToV3Schema

        public static org.apache.avro.Schema writeSimpleAvroFileWithStringToV3Schema​(java.io.File parentDir,
                                                                                     int recordCount,
                                                                                     int recordSizeMin)
                                                                              throws java.io.IOException
        Throws:
        java.io.IOException
      • writeAlternateSimpleAvroFileWithStringToStringSchema

        public static org.apache.avro.Schema writeAlternateSimpleAvroFileWithStringToStringSchema​(java.io.File parentDir)
                                                                                           throws java.io.IOException
        Throws:
        java.io.IOException
      • writeSimpleAvroFileWithStringToStringSchema2

        public static org.apache.avro.Schema writeSimpleAvroFileWithStringToStringSchema2​(java.io.File parentDir)
                                                                                   throws java.io.IOException
        This file overrides half of the value in writeSimpleAvroFileWithStringToStringSchema(File) and add some new values. It's designed to test incremental push
        Throws:
        java.io.IOException
      • writeSimpleAvroFileWithString2StringSchema3

        public static org.apache.avro.Schema writeSimpleAvroFileWithString2StringSchema3​(java.io.File parentDir)
                                                                                  throws java.io.IOException
        This file add some new value in writeSimpleAvroFileWithStringToStringSchema(File) It's designed to test incremental push
        Throws:
        java.io.IOException
      • writeSimpleAvroFileWithDuplicateKey

        public static org.apache.avro.Schema writeSimpleAvroFileWithDuplicateKey​(java.io.File parentDir)
                                                                          throws java.io.IOException
        Throws:
        java.io.IOException
      • writeSimpleAvroFileWithCustomSize

        public static org.apache.avro.Schema writeSimpleAvroFileWithCustomSize​(java.io.File parentDir,
                                                                               int numberOfRecords,
                                                                               int minValueSize,
                                                                               int maxValueSize)
                                                                        throws java.io.IOException
        Throws:
        java.io.IOException
      • writeSimpleAvroFileWithIntToStringSchema

        public static org.apache.avro.Schema writeSimpleAvroFileWithIntToStringSchema​(java.io.File parentDir)
                                                                               throws java.io.IOException
        Throws:
        java.io.IOException
      • writeInvalidAvroFile

        public static void writeInvalidAvroFile​(java.io.File parentDir,
                                                java.lang.String fileName)
                                         throws java.io.IOException
        Throws:
        java.io.IOException
      • writeEmptyAvroFile

        public static org.apache.avro.Schema writeEmptyAvroFile​(java.io.File parentDir,
                                                                org.apache.avro.Schema schema)
                                                         throws java.io.IOException
        Throws:
        java.io.IOException
      • writeEmptyAvroFile

        public static org.apache.avro.Schema writeEmptyAvroFile​(java.io.File parentDir,
                                                                java.lang.String fileName,
                                                                org.apache.avro.Schema schema)
                                                         throws java.io.IOException
        Throws:
        java.io.IOException
      • writeSimpleAvroFileWithStringToNameRecordV1Schema

        public static org.apache.avro.Schema writeSimpleAvroFileWithStringToNameRecordV1Schema​(java.io.File parentDir)
                                                                                        throws java.io.IOException
        Throws:
        java.io.IOException
      • writeSimpleAvroFileWithStringToUserWithStringMapSchema

        public static org.apache.avro.Schema writeSimpleAvroFileWithStringToUserWithStringMapSchema​(java.io.File parentDir,
                                                                                                    int itemsPerRecord)
                                                                                             throws java.io.IOException
        Throws:
        java.io.IOException
      • writeSimpleAvroFileWithStringToPartialUpdateOpRecordSchema

        public static org.apache.avro.Schema writeSimpleAvroFileWithStringToPartialUpdateOpRecordSchema​(java.io.File parentDir)
                                                                                                 throws java.io.IOException
        Throws:
        java.io.IOException
      • writeSimpleAvroFileWithStringToPartialUpdateOpRecordSchema

        public static org.apache.avro.Schema writeSimpleAvroFileWithStringToPartialUpdateOpRecordSchema​(java.io.File parentDir,
                                                                                                        int startIndex,
                                                                                                        int endIndex)
                                                                                                 throws java.io.IOException
        Throws:
        java.io.IOException
      • writeSimpleAvroFileWithASchemaWithAWrongDefaultValue

        public static org.apache.avro.Schema writeSimpleAvroFileWithASchemaWithAWrongDefaultValue​(java.io.File parentDir,
                                                                                                  int numberOfRecords)
                                                                                           throws java.io.IOException
        Throws:
        java.io.IOException
      • writeAvroFileWithManyFloatsAndCustomTotalSize

        public static org.apache.avro.Schema writeAvroFileWithManyFloatsAndCustomTotalSize​(java.io.File parentDir,
                                                                                           int numberOfRecords,
                                                                                           int minValueSize,
                                                                                           int maxValueSize)
                                                                                    throws java.io.IOException
        Throws:
        java.io.IOException
      • getRecordWithFloatArray

        public static org.apache.avro.generic.GenericRecord getRecordWithFloatArray​(org.apache.avro.Schema recordSchema,
                                                                                    int index,
                                                                                    int size)
      • getRecordWithStringMap

        public static org.apache.avro.generic.GenericRecord getRecordWithStringMap​(org.apache.avro.Schema recordSchema,
                                                                                   int index,
                                                                                   int count)
      • writeSimpleVsonFile

        public static KeyAndValueSchemas writeSimpleVsonFile​(java.io.File parentDir)
                                                      throws java.io.IOException
        Throws:
        java.io.IOException
      • writeSchemaWithUnknownFieldIntoAvroFile

        public static org.apache.avro.Schema writeSchemaWithUnknownFieldIntoAvroFile​(java.io.File parentDir)
                                                                              throws java.io.IOException
        Throws:
        java.io.IOException
      • writeVsonByteAndShort

        public static KeyAndValueSchemas writeVsonByteAndShort​(java.io.File parentDir)
                                                        throws java.io.IOException
        Throws:
        java.io.IOException
      • writeComplexVsonFile

        public static KeyAndValueSchemas writeComplexVsonFile​(java.io.File parentDir)
                                                       throws java.io.IOException
        Throws:
        java.io.IOException
      • writeSimpleVsonFileWithUserSchema

        public static Pair<org.apache.avro.Schema,​org.apache.avro.Schema> writeSimpleVsonFileWithUserSchema​(java.io.File parentDir)
                                                                                                           throws java.io.IOException
        Throws:
        java.io.IOException
      • writeMultiLevelVsonFile

        public static KeyAndValueSchemas writeMultiLevelVsonFile​(java.io.File parentDir)
                                                          throws java.io.IOException
        Throws:
        java.io.IOException
      • writeMultiLevelVsonFile2

        public static Pair<VsonSchema,​VsonSchema> writeMultiLevelVsonFile2​(java.io.File parentDir)
                                                                          throws java.io.IOException
        Throws:
        java.io.IOException
      • defaultVPJPropsWithD2Routing

        public static java.util.Properties defaultVPJPropsWithD2Routing​(java.lang.String parentRegionName,
                                                                        java.lang.String parentRegionD2ZkAddress,
                                                                        java.util.Map<java.lang.String,​java.lang.String> childRegionNamesToZkAddress,
                                                                        java.lang.String parentControllerD2ServiceName,
                                                                        java.lang.String childControllerD2ServiceName,
                                                                        java.lang.String inputDirPath,
                                                                        java.lang.String storeName)
      • defaultVPJProps

        public static java.util.Properties defaultVPJProps​(java.lang.String veniceUrl,
                                                           java.lang.String inputDirPath,
                                                           java.lang.String storeName)
      • loadFileAsString

        public static java.lang.String loadFileAsString​(java.lang.String fileName)
      • loadSchemaFileFromResource

        public static java.lang.String loadSchemaFileFromResource​(java.lang.String fileName)
      • loadFileAsStringQuietlyWithErrorLogged

        public static java.lang.String loadFileAsStringQuietlyWithErrorLogged​(java.lang.String fileName)
      • writeSimpleAvroFileForValidateSchemaAndBuildDictMapperOutput

        public static org.apache.avro.Schema writeSimpleAvroFileForValidateSchemaAndBuildDictMapperOutput​(java.io.File parentDir,
                                                                                                          java.lang.String file,
                                                                                                          long inputFileDataSize,
                                                                                                          java.nio.ByteBuffer zstdDictionary,
                                                                                                          org.apache.avro.Schema avroSchema)
                                                                                                   throws java.io.IOException
        Throws:
        java.io.IOException
      • writeETLFileWithUserSchema

        public static org.apache.avro.Schema writeETLFileWithUserSchema​(java.io.File parentDir)
                                                                 throws java.io.IOException
        Throws:
        java.io.IOException
      • writeETLFileWithUserSchemaAndNullDefaultValue

        public static org.apache.avro.Schema writeETLFileWithUserSchemaAndNullDefaultValue​(java.io.File parentDir)
                                                                                    throws java.io.IOException
        Throws:
        java.io.IOException
      • writeETLFileWithUnionWithNullSchema

        public static org.apache.avro.Schema writeETLFileWithUnionWithNullSchema​(java.io.File parentDir)
                                                                          throws java.io.IOException
        Throws:
        java.io.IOException
      • writeETLFileWithUnionWithoutNullSchema

        public static org.apache.avro.Schema writeETLFileWithUnionWithoutNullSchema​(java.io.File parentDir)
                                                                             throws java.io.IOException
        Throws:
        java.io.IOException
      • getETLFileSchema

        public static org.apache.avro.Schema getETLFileSchema​(org.apache.avro.Schema keySchema,
                                                              org.apache.avro.Schema valueSchema)
      • getETLFileSchemaWithNullDefaultValue

        public static org.apache.avro.Schema getETLFileSchemaWithNullDefaultValue​(org.apache.avro.Schema keySchema,
                                                                                  org.apache.avro.Schema valueSchema)
      • runPushJob

        public static void runPushJob​(java.lang.String jobId,
                                      java.util.Properties props)
      • runPushJob

        public static void runPushJob​(java.lang.String jobId,
                                      java.util.Properties props,
                                      java.util.function.Consumer<VenicePushJob> jobTransformer)