HDFS 作為開源界比較成熟的分布式文件存儲系統，適用于海量文件存儲，本文介紹了如何使用 Java 操作 HDFS，采用 Maven 管理包。

pom.xml

<dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-client</artifactId><version>2.6.0</version></dependency> <dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-common</artifactId><version>2.6.0</version></dependency> <dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-hdfs</artifactId><version>2.6.0</version> </dependency>

HadoopUtil.java

import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import java.net.URI;/*** Hadoop工具類*/ public class HadoopUtil {private static String hdfsPath="hdfs://192.168.91.143:8020";private static String hdfsName="hdfs";/*** 獲取HDFS配置信息* @return*/private static Configuration getConfiguration() {Configuration configuration = new Configuration();configuration.set("fs.defaultFS", hdfsPath);return configuration;}/*** 獲取HDFS文件系統對象* @return* @throws Exception*/public static FileSystem getFileSystem() throws Exception {// 客戶端去操作hdfs時是有一個用戶身份的，默認情況下hdfs客戶端api會從jvm中獲取一個參數作為自己的用戶身份 DHADOOP_USER_NAME=hadoop // FileSystem hdfs = FileSystem.get(getHdfsConfig()); //默認獲取 // 也可以在構造客戶端fs對象時，通過參數傳遞進去FileSystem fileSystem = FileSystem.get(new URI(hdfsPath), getConfiguration(), hdfsName);return fileSystem;} }

HDFSApp.java

import org.apache.commons.compress.utils.IOUtils; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path;import java.io.FileInputStream; import java.io.InputStream;public class HDFSApp {public static void main(String[] args) throws Exception {FileSystem fs= HadoopUtil.getFileSystem();//目錄不存在，創建String path="/javatest";Path dirPath=new Path(path);if(!fs.exists(dirPath)) {boolean isOk = fs.mkdirs(dirPath);fs.close();if (isOk) {System.out.println("create dir success");} else {System.out.println("create dir fail");}}// 上傳時默認當前目錄，后面自動拼接文件的目錄String fileName="text.txt";Path filePath = new Path(path + "/" + fileName);// 打開一個輸出流FSDataOutputStream outputStream=null;FileInputStream in=null;try {outputStream = fs.create(filePath);in = new FileInputStream("e:/temp/text.txt");IOUtils.copy(in, outputStream, 4096);}finally {in.close();outputStream.close();}//直接從本地拷貝文件fs.copyFromLocalFile(new Path("e:/temp/text.txt"),new Path(path + "/textcopy.txt" ));//直接下載文件到本地fs.copyToLocalFile(new Path(path + "/textcopy.txt" ),new Path("e:/temp/textcopy.txt"));//文件改名fs.rename(new Path(path + "/textcopy.txt" ),new Path(path + "/textrename.txt" ));//下載文件InputStream inputStream =null;try {inputStream = fs.open(filePath);IOUtils.copy(inputStream, System.out, 4096);}finally {inputStream.close();}//刪除文件boolean isOk = fs.deleteOnExit(filePath);fs.close();if (isOk) {System.out.println("delete file success");} else {System.out.println("delete file fail");}fs.close();} }

?

總結

以上是生活随笔為你收集整理的Java 操作 HDFS的全部內容，希望文章能夠幫你解決所遇到的問題。

如果覺得生活随笔網站內容還不錯，歡迎將生活随笔推薦給好友。