Hive教程（06）- Hive SerDe序列化与反序列化

CREATE [EXTERNAL] TABLE [IF NOT EXISTS] table_name
[(col_name data_type [COMMENT col_comment], ...)]
[COMMENT table_comment]
[PARTITIONED BY (col_name data_type [COMMENT col_comment], ...)]
[CLUSTERED BY (col_name, col_name, ...)
[SORTED BY (col_name [ASC|DESC], ...)]
INTO num_buckets BUCKETS]
[ROW FORMAT row_format]
[STORED AS file_format]
[LOCATION hdfs_path]

ALTER TABLE person SET SERDEPROPERTIES (‘serialization.encoding’=’GBK’)

public class MySerDe extends AbstractSerDe {
    // params
    private List<String> columnNames = null;
    private List<TypeInfo> columnTypes = null;
    private ObjectInspector objectInspector = null;
    // seperator
    private String nullString = null;
    private String lineSep = null;
    private String kvSep = null;
    @Override
    public void initialize(Configuration conf, Properties tbl)
            throws SerDeException {
        // Read sep
        lineSep = "\n";
        kvSep = "=";
        nullString = tbl.getProperty(Constants.SERIALIZATION_NULL_FORMAT, "");
        // Read Column Names
        String columnNameProp = tbl.getProperty(Constants.LIST_COLUMNS);
        if (columnNameProp != null && columnNameProp.length() > 0) {
            columnNames = Arrays.asList(columnNameProp.split(","));
        } else {
            columnNames = new ArrayList<String>();
        }
        // Read Column Types
        String columnTypeProp = tbl.getProperty(Constants.LIST_COLUMN_TYPES);
        // default all string
        if (columnTypeProp == null) {
            String[] types = new String[columnNames.size()];
            Arrays.fill(types, 0, types.length, Constants.STRING_TYPE_NAME);
            columnTypeProp = StringUtils.join(types, ":");
        }
        columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProp);
        // Check column and types equals
        if (columnTypes.size() != columnNames.size()) {
            throw new SerDeException("len(columnNames) != len(columntTypes)");
        }
        // Create ObjectInspectors from the type information for each column
        List<ObjectInspector> columnOIs = new ArrayList<ObjectInspector>();
        ObjectInspector oi;
        for (int c = 0; c < columnNames.size(); c++) {
            oi = TypeInfoUtils
                    .getStandardJavaObjectInspectorFromTypeInfo(columnTypes
                            .get(c));
            columnOIs.add(oi);
        }
        objectInspector = ObjectInspectorFactory
                .getStandardStructObjectInspector(columnNames, columnOIs);
    }
    @Override
    public Object deserialize(Writable wr) throws SerDeException {
        // Split to kv pair
        if (wr == null)
            return null;
        Map<String, String> kvMap = new HashMap<String, String>();
        Text text = (Text) wr;
        for (String kv : text.toString().split(lineSep)) {
            String[] pair = kv.split(kvSep);
            if (pair.length == 2) {
                kvMap.put(pair[0], pair[1]);
            }
        }
        // Set according to col_names and col_types
        ArrayList<Object> row = new ArrayList<Object>();
        String colName = null;
        TypeInfo type_info = null;
        Object obj = null;
        for (int i = 0; i < columnNames.size(); i++) {
            colName = columnNames.get(i);
            type_info = columnTypes.get(i);
            obj = null;
            if (type_info.getCategory() == ObjectInspector.Category.PRIMITIVE) {
                PrimitiveTypeInfo p_type_info = (PrimitiveTypeInfo) type_info;
                switch (p_type_info.getPrimitiveCategory()) {
                case STRING:
                    obj = StringUtils.defaultString(kvMap.get(colName), "");
                    break;
                case LONG:
                case INT:
                    try {
                        obj = Long.parseLong(kvMap.get(colName));
                    } catch (Exception e) {
                    }
                }
            }
            row.add(obj);
        }
        return row;
    }
    @Override
    public ObjectInspector getObjectInspector() throws SerDeException {
        return objectInspector;
    }
    @Override
    public SerDeStats getSerDeStats() {
        return null;
    }
    @Override
    public Class<? extends Writable> getSerializedClass() {
        return Text.class;
    }
    @Override
    public Writable serialize(Object arg0, ObjectInspector arg1)
            throws SerDeException {
        return null;
    }
}

hive > add jar MySerDe.jar

CREATE EXTERNAL TABLE IF NOT EXISTS teacher ( 
          id BIGINT, 
          name STRING,
          age INT)
ROW FORMAT SERDE 'com.coder4.hive.MySerDe'
STORED AS TEXTFILE
LOCATION '/usr/hive/text/'