一、avro是一個數據序列化框架,可以高效得進行序列化和反序列化,支持C, C++, C#, java, php, Python, 和Ruby語言。現在使用Java來讀寫。
二、環境搭建
1、下載avro-1.7.7.jar and avro-tools-1.7.7.jar兩個jar包,放到指定文件目錄。下載地址 http://www.trieuvan.com/apache/avro/avro-1.7.7/java/
我放到了D:/soft/avro 文件夾,在改目錄下新建java文件夾,用來存放生成的Java代碼
2、該目錄下新建user.avsc文件,內容是:
{"namespace": "example.avro", "type": "record", "name": "User", "fields": [ {"name": "name", "type": "string"}, {"name": "favorite_number", "type": ["int", "null"]}, {"name": "favorite_color", "type": ["string", "null"]}
]}
3、打開cmd,進入到該目錄,執行命令生成User類,注意命令后面有個".",表示生成的代碼放在本目錄下。
java -jar avro-tools-1.7.7.jar compile schema user.avsc java .
在該文件夾下的Java文件下的../example/avro/目錄下就會生成User.java文件。
4.使用eclipse新建maven項目,在pom.xml加入avro的依賴。
<dependency> <groupId>org.apache.avro</groupId> <artifactId>avro</artifactId> <version>1.7.7</version></dependency>
三、生成的User.java文件的內容如下。
把生成的User.java類復制到工程中,注意這個User.java里面生成的User類及其內部類的包名默認是user.avsc文件中的namespace的值,
在本例中也就是example.avro。需要全部替換為自己的包名。
最簡單的方法就是把User.java中的example.avro全部替換為自己的包名。
/** * Autogenerated by Avro * * DO NOT EDIT DIRECTLY */package example.avro; @Sup)@org.apache.avro.specific.AvroGeneratedpublic class User extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord { public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{/"type/":/"record/",/"name/":/"User/",/"namespace/":/"example.avro/",/"fields/":[{/"name/":/"name/",/"type/":/"string/"},{/"name/":/"favorite_number/",/"type/":[/"int/",/"null/"]},{/"name/":/"favorite_color/",/"type/":[/"string/",/"null/"]}]}"); public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; } @Deprecated public java.lang.CharSequence name; @Deprecated public java.lang.Integer favorite_number; @Deprecated public java.lang.CharSequence favorite_color; /** * Default constructor. Note that this does not initialize fields * to their default values from the schema. If that is desired then * one should use <code>newBuilder()</code>. */ public User() {} /** * All-args constructor. */ public User(java.lang.CharSequence name, java.lang.Integer favorite_number, java.lang.CharSequence favorite_color) { this.name = name; this.favorite_number = favorite_number; this.favorite_color = favorite_color; } public org.apache.avro.Schema getSchema() { return SCHEMA$; } // Used by DatumWriter. applications should not call. public java.lang.Object get(int field$) { switch (field$) { case 0: return name; case 1: return favorite_number; case 2: return favorite_color; default: throw new org.apache.avro.AvroRuntimeException("Bad index"); } } // Used by DatumReader. Applications should not call. @SuppressWarnings(value="unchecked") public void put(int field$, java.lang.Object value$) { switch (field$) { case 0: name = (java.lang.CharSequence)value$; break; case 1: favorite_number = (java.lang.Integer)value$; break; case 2: favorite_color = (java.lang.CharSequence)value$; break; default: throw new org.apache.avro.AvroRuntimeException("Bad index"); } } /** * Gets the value of the 'name' field. */ public java.lang.CharSequence getName() { return name; } /** * Sets the value of the 'name' field. * @param value the value to set. */ public void setName(java.lang.CharSequence value) { this.name = value; } /** * Gets the value of the 'favorite_number' field. */ public java.lang.Integer getFavoriteNumber() { return favorite_number; } /** * Sets the value of the 'favorite_number' field. * @param value the value to set. */ public void setFavoriteNumber(java.lang.Integer value) { this.favorite_number = value; } /** * Gets the value of the 'favorite_color' field. */ public java.lang.CharSequence getFavoriteColor() { return favorite_color; } /** * Sets the value of the 'favorite_color' field. * @param value the value to set. */ public void setFavoriteColor(java.lang.CharSequence value) { this.favorite_color = value; } /** Creates a new User RecordBuilder */ public static example.avro.User.Builder newBuilder() { return new example.avro.User.Builder(); } /** Creates a new User RecordBuilder by copying an existing Builder */ public static example.avro.User.Builder newBuilder(example.avro.User.Builder other) { return new example.avro.User.Builder(other); } /** Creates a new User RecordBuilder by copying an existing User instance */ public static example.avro.User.Builder newBuilder(example.avro.User other) { return new example.avro.User.Builder(other); } /** * RecordBuilder for User instances. */ public static class Builder extends org.apache.avro.specific.SpecificRecordBuilderBase<User> implements org.apache.avro.data.RecordBuilder<User> { private java.lang.CharSequence name; private java.lang.Integer favorite_number; private java.lang.CharSequence favorite_color; /** Creates a new Builder */ private Builder() { super(example.avro.User.SCHEMA$); } /** Creates a Builder by copying an existing Builder */ private Builder(example.avro.User.Builder other) { super(other); if (isValidValue(fields()[0], other.name)) { this.name = data().deepCopy(fields()[0].schema(), other.name); fieldSetFlags()[0] = true; } if (isValidValue(fields()[1], other.favorite_number)) { this.favorite_number = data().deepCopy(fields()[1].schema(), other.favorite_number); fieldSetFlags()[1] = true; } if (isValidValue(fields()[2], other.favorite_color)) { this.favorite_color = data().deepCopy(fields()[2].schema(), other.favorite_color); fieldSetFlags()[2] = true; } } /** Creates a Builder by copying an existing User instance */ private Builder(example.avro.User other) { super(example.avro.User.SCHEMA$); if (isValidValue(fields()[0], other.name)) { this.name = data().deepCopy(fields()[0].schema(), other.name); fieldSetFlags()[0] = true; } if (isValidValue(fields()[1], other.favorite_number)) { this.favorite_number = data().deepCopy(fields()[1].schema(), other.favorite_number); fieldSetFlags()[1] = true; } if (isValidValue(fields()[2], other.favorite_color)) { this.favorite_color = data().deepCopy(fields()[2].schema(), other.favorite_color); fieldSetFlags()[2] = true; } } /** Gets the value of the 'name' field */ public java.lang.CharSequence getName() { return name; } /** Sets the value of the 'name' field */ public example.avro.User.Builder setName(java.lang.CharSequence value) { validate(fields()[0], value); this.name = value; fieldSetFlags()[0] = true; return this; } /** Checks whether the 'name' field has been set */ public boolean hasName() { return fieldSetFlags()[0]; } /** Clears the value of the 'name' field */ public example.avro.User.Builder clearName() { name = null; fieldSetFlags()[0] = false; return this; } /** Gets the value of the 'favorite_number' field */ public java.lang.Integer getFavoriteNumber() { return favorite_number; } /** Sets the value of the 'favorite_number' field */ public example.avro.User.Builder setFavoriteNumber(java.lang.Integer value) { validate(fields()[1], value); this.favorite_number = value; fieldSetFlags()[1] = true; return this; } /** Checks whether the 'favorite_number' field has been set */ public boolean hasFavoriteNumber() { return fieldSetFlags()[1]; } /** Clears the value of the 'favorite_number' field */ public example.avro.User.Builder clearFavoriteNumber() { favorite_number = null; fieldSetFlags()[1] = false; return this; } /** Gets the value of the 'favorite_color' field */ public java.lang.CharSequence getFavoriteColor() { return favorite_color; } /** Sets the value of the 'favorite_color' field */ public example.avro.User.Builder setFavoriteColor(java.lang.CharSequence value) { validate(fields()[2], value); this.favorite_color = value; fieldSetFlags()[2] = true; return this; } /** Checks whether the 'favorite_color' field has been set */ public boolean hasFavoriteColor() { return fieldSetFlags()[2]; } /** Clears the value of the 'favorite_color' field */ public example.avro.User.Builder clearFavoriteColor() { favorite_color = null; fieldSetFlags()[2] = false; return this; } @Override public User build() { try { User record = new User(); record.name = fieldSetFlags()[0] ? this.name : (java.lang.CharSequence) defaultValue(fields()[0]); record.favorite_number = fieldSetFlags()[1] ? this.favorite_number : (java.lang.Integer) defaultValue(fields()[1]); record.favorite_color = fieldSetFlags()[2] ? this.favorite_color : (java.lang.CharSequence) defaultValue(fields()[2]); return record; } catch (Exception e) { throw new org.apache.avro.AvroRuntimeException(e); } } }}
四、用Java實現序列化,即寫avro文件。
新建一個Java類
public static void main(String[] args) throws IOException {
// 聲明并初始化User對象
// 方式一
User user1 = new User(); user1.setName("zhangsan"); user1.setFavoriteNumber(21); user1.setFavoriteColor(null);
// 方式二 使用構造函數 // Alternate constructor User user2 = new User("Ben", 7, "red");
// 方式三,使用Build方式 // Construct via builder User user3 = User.newBuilder() .setName("Charlie") .setFavoriteColor("blue") .setFavoriteNumber(null) .build(); String path = "D://tmp//user.avro"; // avro文件存放目錄 DatumWriter<User> userDatumWriter = new SpecificDatumWriter<User>(User.class); DataFileWriter<User> dataFileWriter = new DataFileWriter<User>(userDatumWriter); dataFileWriter.create(user1.getSchema(), new File(path));
// 把生成的user對象寫入到avro文件 dataFileWriter.append(user1); dataFileWriter.append(user2); dataFileWriter.append(user3); dataFileWriter.close();
}
run一下代碼,查看指定文件目錄是否生成了avro文件。
五、Java讀取avro文件,即實現avro反序列化。
public static void main(String[] args) throws IOException { DatumReader<User> reader = new SpecificDatumReader<User>(User.class); DataFileReader<User> dataFileReader = new DataFileReader<User>(new File("D://tmp//user.avro"), reader); User user = null; while (dataFileReader.hasNext()) { user = dataFileReader.next(); System.out.println(user); } }
運行結果:
{"name": "zhangsan", "favorite_number": 21, "favorite_color": null}{"name": "Ben", "favorite_number": 7, "favorite_color": "red"}{"name": "Charlie", "favorite_number": null, "favorite_color": "blue"}
至此,例子寫完。
新聞熱點
疑難解答