Java Hadoop分布式系統文件操作

Java Hadoop分布式系統文件操作

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

/**
*
* The utilities to operate file on hadoop hdfs.
*
* @author luolihui 2011-07-18
*
*/
public class DFSOperator {

private static final String ROOT_PATH = "hdfs:///";
private static final int BUFFER_SIZE = 4096;

/**
* construct.
*/
public DFSOperator(){}

/**
* Create a file on hdfs.The root path is /.<br>
* for example: DFSOperator.createFile("/lory/test1.txt", true);
* @param path the file name to open
* @param overwrite if a file with this name already exists, then if true, the file will be
* @return true if delete is successful else IOException.
* @throws IOException
*/
public static boolean createFile(String path, boolean overwrite) throws IOException
{
//String uri = "hdfs://192.168.1.100:9000";
//FileSystem fs1 = FileSystem.get(URI.create(uri), conf);

Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
Path f = new Path(ROOT_PATH + path);
fs.create(f, overwrite);
fs.close();
return true;
}

/**
* Delete a file on hdfs.The root path is /. <br>
* for example: DFSOperator.deleteFile("/user/hadoop/output", true);
* @param path the path to delete
* @param recursive if path is a directory and set to true, the directory is deleted else throws an exception. In case of a file the recursive can be set to either true or false.
* @return true if delete is successful else IOException.
* @throws IOException
*/
public static boolean deleteFile(String path, boolean recursive) throws IOException
{
//String uri = "hdfs://192.168.1.100:9000";
//FileSystem fs1 = FileSystem.get(URI.create(uri), conf);

Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
Path f = new Path(ROOT_PATH + path);
fs.delete(f, recursive);
fs.close();
return true;
}

/**
* Read a file to string on hadoop hdfs. From stream to string. <br>
* for example: System.out.println(DFSOperator.readDFSFileToString("/user/hadoop/input/test3.txt"));
* @param path the path to read
* @return true if read is successful else IOException.
* @throws IOException
*/
public static String readDFSFileToString(String path) throws IOException
{
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
Path f = new Path(ROOT_PATH + path);
InputStream in = null;
String str = null;
StringBuilder sb = new StringBuilder(BUFFER_SIZE);
if (fs.exists(f))
{
in = fs.open(f);
BufferedReader bf = new BufferedReader(new InputStreamReader(in));

while ((str = bf.readLine()) != null)
{
sb.append(str);
sb.append("\n");
}

in.close();
bf.close();
fs.close();
return sb.toString();
}
else
{
return null;
}

}
/**
* Write string to a hadoop hdfs file. <br>
* for example: DFSOperator.writeStringToDFSFile("/lory/test1.txt", "You are a bad man.\nReally!\n");
* @param path the file where the string to write in.
* @param string the context to write in a file.
* @return true if write is successful else IOException.
* @throws IOException
*/
public static boolean writeStringToDFSFile(String path, String string) throws IOException
{
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
FSDataOutputStream os = null;
Path f = new Path(ROOT_PATH + path);
os = fs.create(f,true);
os.writeBytes(string);

os.close();
fs.close();
return true;
}

public static void main(String[] args)
{
try {
DFSOperator.createFile("/lory/test1.txt", true);
DFSOperator.deleteFile("/dfs_operator.txt", true);
DFSOperator.writeStringToDFSFile("/lory/test1.txt", "You are a bad man.\nReally?\n");
System.out.println(DFSOperator.readDFSFileToString("/lory/test1.txt"));
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
System.out.println("===end===");
}
}

--------------------------------------分割線 --------------------------------------

Ubuntu 13.04上搭建Hadoop環境 http://www.linuxidc.com/Linux/2013-06/86106.htm

Ubuntu 12.10 +Hadoop 1.2.1版本集群配置 http://www.linuxidc.com/Linux/2013-09/90600.htm

Ubuntu上搭建Hadoop環境（單機模式+偽分布模式） http://www.linuxidc.com/Linux/2013-01/77681.htm

Ubuntu下Hadoop環境的配置 http://www.linuxidc.com/Linux/2012-11/74539.htm

單機版搭建Hadoop環境圖文教程詳解 http://www.linuxidc.com/Linux/2012-02/53927.htm

--------------------------------------分割線 --------------------------------------

更多Hadoop相關信息見Hadoop 專題頁面 http://www.linuxidc.com/topicnews.aspx?tid=13