Hadoop 2

2013-07-25

Hadoop
remote
procedure callRPCRPC
RPC

HDFSMapReduce Hadoop
Java
org.apache.hadoop.io Writable
Writable DataOutput
DataInput
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
*
http://www.apache.org/licenses/LICENSE-2.0
1

*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.io;
import java.io.DataOutput;
import java.io.DataInput;
import java.io.IOException;
/**
* A serializable object which implements a simple, efficient, serialization
* protocol, based on {@link DataInput} and {@link DataOutput}.
*
* <p>Any <code>key</code> or <code>value</code> type in the Hadoop Map-Reduce
* framework implements this interface.</p>
*
* <p>Implementations typically implement a static <code>read(DataInput)</code>
* method which constructs a new instance, calls {@link #readFields(DataInput)}
* and returns the instance.</p>
*
* <p>Example:</p>
* <p><blockquote><pre>
*
public class MyWritable implements Writable {
*
// Some data
*
private int counter;
*
private long timestamp;
*
*
public void write(DataOutput out) throws IOException {
*
out.writeInt(counter);
*
out.writeLong(timestamp);
*
}
*
*
public void readFields(DataInput in) throws IOException {
*
counter = in.readInt();
*
timestamp = in.readLong();
*
}
*
*
public static MyWritable read(DataInput in) throws IOException {
*
MyWritable w = new MyWritable();
2

*
w.readFields(in);
*
return w;
*
}
*
}
* </pre></blockquote></p>
*/
public interface Writable {
/**
* Serialize the fields of this object to <code>out</code>.
*
* @param out <code>DataOuput</code> to serialize this object into.
* @throws IOException
*/
void write(DataOutput out) throws IOException;
/**
* Deserialize the fields of this object from <code>in</code>.
*
* <p>For efficiency, implementations should attempt to re-use storage in the
* existing object where possible.</p>
*
* @param in <code>DataInput</code> to deseriablize this object from.
* @throws IOException
*/
void readFields(DataInput in) throws IOException;
}

write readFields
Writable
org.apache.hadoop.io

WritableComparable Writable java.lang.Comparable

IntWritable WritableComparable
MapReduce

Hadoop Java Comparator RawComparator

package org.apache.hadoop.io;
import java.util.Comparator;
import org.apache.hadoop.io.serializer.DeserializerComparator;
/**
* <p>
4

* A {@link Comparator} that operates directly on byte representations of

* objects.
* </p>
* @param <T>
* @see DeserializerComparator
*/
public interface RawComparator<T> extends Comparator<T> {
public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2);
}

WritableComparator WritableComparable RawComparator

compare()
RawComparator Writable

Hadoop org.apache.hadoop.io Writable

Writable Java short char

IntWritable get() set()
Java

Writable

boolean

BooleanWritable

byte

ByteWritable

int

IntWritable

VIntWritable

1-5

FloatWritable

float

long

double

LongWritable

VLongWritable

1-9

DoubleWritable

IntWritable LongWritable
VIntWritable VLongWritable-127
127 -127 127

Text UTF-8 Writable java.lang.String

Writable
ObjectWritable Hadoop
RPC RPC Java String Writable
ObjectWritable
RPC
/ Writable Writable
RPC MyWritable ObjectWritable
ObjectWritable

ObjectWritable
ObjectWritable WritableFactories Writable
MyWritable WritableFactories
WritableFactories.setFactory

MapReduce Writable
MapReduce API

Hadoop serialization
framework API Serialization
WritableSerialization Writable Serialization Serialization
Serializer Deserializer

Hadoop io.serizalizations Serialization

org.apache.hadoop.io.serializer. WritableSerialization