编程实现以下功能，并利用Hadoop提供的Shell命令完成相同任务：
- 向HDFS中上传任意文本文件，如果指定的文件在HDFS中已经存在，则由用户来指定是追加到原有文件末尾还是覆盖原有的文件；
- 从HDFS中下载指定文件，如果本地文件与要下载的文件名称相同，则自动对下载的文件重命名；
- 将HDFS中指定文件的内容输出到终端中；
- 显示HDFS中指定的文件的读写权限、大小、创建时间、路径等信息；
- 给定HDFS中某一个目录，输出该目录下的所有文件的读写权限、大小、创建时间、路径等信息，如果该文件是目录，则递归输出该目录下所有文件相关信息；
- 提供一个HDFS内的文件的路径，对该文件进行创建和删除操作。如果文件所在目录不存在，则自动创建目录；
- 提供一个HDFS的目录的路径，对该目录进行创建和删除操作。创建目录时，如果目录文件所在目录不存在，则自动创建相应目录；删除目录时，由用户指定当该目录不为空时是否还删除该目录；
- 向HDFS中指定的文件追加内容，由用户指定内容追加到原有文件的开头或结尾；
- 删除HDFS中指定的文件；
- 在HDFS中，将文件从源路径移动到目的路径。
首先使用编程实现
pom.xml
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"><modelVersion>4.0.0</modelVersion><groupId>com.example</groupId><artifactId>hdfsoperate</artifactId><version>1.0-SNAPSHOT</version><name>hdfsoperate</name><properties><project.build.sourceEncoding>UTF-8</project.build.sourceEncoding><maven.compiler.target>1.8</maven.compiler.target><maven.compiler.source>1.8</maven.compiler.source><junit.version>5.9.2</junit.version></properties><dependencies><!-- Hadoop Dependencies --><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-common</artifactId><version>3.3.0</version></dependency><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-hdfs</artifactId><version>3.3.0</version></dependency><dependency><groupId>org.apache.hadoop</groupId><artifactId>hadoop-client</artifactId><version>3.3.0</version></dependency><!-- Testing Dependencies --><dependency><groupId>org.junit.jupiter</groupId><artifactId>junit-jupiter-api</artifactId><version>${junit.version}</version><scope>test</scope></dependency><dependency><groupId>org.junit.jupiter</groupId><artifactId>junit-jupiter-engine</artifactId><version>${junit.version}</version><scope>test</scope></dependency></dependencies><build><plugins></plugins></build>
</project>HDFSUtil.java
package com.example.hdfs;import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.IOUtils;import java.io.*;
import java.net.URI;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;/*** HDFS操作工具类，提供各种HDFS文件系统操作功能*/
public class HDFSUtil {private final FileSystem fs;private final Configuration conf;/*** 构造函数，初始化HDFS连接* @param hdfsUri HDFS的URI，例如：hdfs://192.168.88.151:8020* @throws IOException IO异常*/public HDFSUtil(String hdfsUri) throws IOException {conf = new Configuration();System.setProperty("HADOOP_USER_NAME", "root");fs = FileSystem.get(URI.create(hdfsUri), conf);}/*** 关闭HDFS连接* @throws IOException IO异常*/public void close() throws IOException {if (fs != null) {fs.close();}}/*** 获取FileSystem实例* @return FileSystem实例*/public FileSystem getFileSystem() {return fs;}/*** 向HDFS中上传文本文件，支持追加或覆盖* @param localFilePath 本地文件路径* @param hdfsFilePath HDFS文件路径* @param append 是否追加（当文件存在时）* @throws IOException IO异常*/public void uploadFile(String localFilePath, String hdfsFilePath, boolean append) throws IOException {Path localPath = new Path(localFilePath);Path hdfsPath = new Path(hdfsFilePath);// 检查文件是否存在if (fs.exists(hdfsPath)) {if (append) {// 追加模式FSDataOutputStream outputStream = fs.append(hdfsPath);FileInputStream inputStream = new FileInputStream(localFilePath);IOUtils.copyBytes(inputStream, outputStream, conf);inputStream.close();outputStream.close();System.out.println("文件已追加到: " + hdfsFilePath);} else {// 覆盖模式fs.copyFromLocalFile(localPath, hdfsPath);System.out.println("文件已覆盖: " + hdfsFilePath);}} else {// 确保父目录存在Path parentPath = hdfsPath.getParent();if (parentPath != null && !fs.exists(parentPath)) {fs.mkdirs(parentPath);}// 上传新文件fs.copyFromLocalFile(localPath, hdfsPath);System.out.println("文件已上传: " + hdfsFilePath);}}/*** 从HDFS下载文件，如果本地文件已存在则自动重命名* @param hdfsFilePath HDFS文件路径* @param localDir 本地目录路径* @throws IOException IO异常*/public void downloadFile(String hdfsFilePath, String localDir) throws IOException {Path hdfsPath = new Path(hdfsFilePath);FileStatus status = fs.getFileStatus(hdfsPath);String fileName = status.getPath().getName();// 构建本地文件路径File localFile = new File(localDir, fileName);// 检查本地文件是否存在，如果存在则重命名int count = 1;String baseName = fileName;String extension = "";int dotIndex = fileName.lastIndexOf('.');if (dotIndex > 0) {baseName = fileName.substring(0, dotIndex);extension = fileName.substring(dotIndex);}while (localFile.exists()) {fileName = baseName + "_" + count + extension;localFile = new File(localDir, fileName);count++;}// 下载文件Path localPath = new Path(localFile.getAbsolutePath());fs.copyToLocalFile(hdfsPath, localPath);System.out.println("文件已下载到: " + localFile.getAbsolutePath());}/*** 将HDFS文件内容输出到终端* @param hdfsFilePath HDFS文件路径* @throws IOException IO异常*/public void catFile(String hdfsFilePath) throws IOException {Path hdfsPath = new Path(hdfsFilePath);try (FSDataInputStream inputStream = fs.open(hdfsPath)) {IOUtils.copyBytes(inputStream, System.out, conf, false);System.out.flush(); // 确保输出被刷新}}/*** 显示HDFS文件的详细信息* @param hdfsFilePath HDFS文件路径* @throws IOException IO异常*/public void lsFile(String hdfsFilePath) throws IOException {Path hdfsPath = new Path(hdfsFilePath);FileStatus status = fs.getFileStatus(hdfsPath);printFileStatus(status);}/*** 递归显示目录下所有文件的详细信息* @param hdfsDirPath HDFS目录路径* @throws IOException IO异常*/public void lsRecursive(String hdfsDirPath) throws IOException {Path hdfsPath = new Path(hdfsDirPath);FileStatus status = fs.getFileStatus(hdfsPath);if (status.isDirectory()) {List<FileStatus> allStatus = new ArrayList<>();listRecursive(hdfsPath, allStatus);for (FileStatus fileStatus : allStatus) {printFileStatus(fileStatus);}} else {printFileStatus(status);}}/*** 递归列出目录下所有文件* @param path 路径* @param statusList 文件状态列表* @throws IOException IO异常*/private void listRecursive(Path path, List<FileStatus> statusList) throws IOException {FileStatus[] statuses = fs.listStatus(path);for (FileStatus status : statuses) {statusList.add(status);if (status.isDirectory()) {listRecursive(status.getPath(), statusList);}}}/*** 打印文件状态信息* @param status 文件状态*/private void printFileStatus(FileStatus status) {SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");FsPermission permission = status.getPermission();String type = status.isDirectory() ? "d" : "-";String owner = status.getOwner();String group = status.getGroup();long length = status.getLen();String modifyTime = dateFormat.format(new Date(status.getModificationTime()));String path = status.getPath().toString();System.out.printf("%s%s %s %s %10d %s %s\n", type, permission, owner, group, length, modifyTime, path);}/*** 创建或删除HDFS文件* @param hdfsFilePath HDFS文件路径* @param create 是否创建（true创建，false删除）* @throws IOException IO异常*/public void fileOperation(String hdfsFilePath, boolean create) throws IOException {Path hdfsPath = new Path(hdfsFilePath);if (create) {// 创建文件，确保父目录存在Path parentPath = hdfsPath.getParent();if (parentPath != null && !fs.exists(parentPath)) {fs.mkdirs(parentPath);}if (fs.exists(hdfsPath)) {System.out.println("文件已存在: " + hdfsFilePath);} else {FSDataOutputStream outputStream = fs.create(hdfsPath);outputStream.close();System.out.println("文件已创建: " + hdfsFilePath);}} else {// 删除文件if (fs.exists(hdfsPath)) {boolean success = fs.delete(hdfsPath, false);if (success) {System.out.println("文件已删除: " + hdfsFilePath);} else {System.out.println("删除文件失败: " + hdfsFilePath);}} else {System.out.println("文件不存在: " + hdfsFilePath);}}}/*** 创建或删除HDFS目录* @param hdfsDirPath HDFS目录路径* @param create 是否创建（true创建，false删除）* @param forceDelete 是否强制删除非空目录* @throws IOException IO异常*/public void dirOperation(String hdfsDirPath, boolean create, boolean forceDelete) throws IOException {Path hdfsPath = new Path(hdfsDirPath);if (create) {// 创建目录（自动创建父目录）boolean success = fs.mkdirs(hdfsPath);if (success) {System.out.println("目录已创建: " + hdfsDirPath);} else {System.out.println("创建目录失败: " + hdfsDirPath);}} else {// 删除目录if (fs.exists(hdfsPath)) {if (fs.isDirectory(hdfsPath)) {// 检查目录是否为空FileStatus[] statuses = fs.listStatus(hdfsPath);if (statuses.length > 0 && !forceDelete) {System.out.println("目录不为空，无法删除: " + hdfsDirPath);return;}boolean success = fs.delete(hdfsPath, true);if (success) {System.out.println("目录已删除: " + hdfsDirPath);} else {System.out.println("删除目录失败: " + hdfsDirPath);}} else {System.out.println("指定路径不是目录: " + hdfsDirPath);}} else {System.out.println("目录不存在: " + hdfsDirPath);}}}/*** 向文件追加内容（开头或结尾）* @param hdfsFilePath HDFS文件路径* @param content 要追加的内容* @param appendToStart 是否追加到开头* @throws IOException IO异常*/public void appendToFile(String hdfsFilePath, String content, boolean appendToStart) throws IOException {Path hdfsPath = new Path(hdfsFilePath);if (!fs.exists(hdfsPath)) {System.out.println("文件不存在: " + hdfsFilePath);return;}if (appendToStart) {// 追加到开头需要重写整个文件// 创建临时文件Path tempPath = new Path(hdfsFilePath + ".tmp");FSDataOutputStream tempOutput = fs.create(tempPath);// 写入新内容tempOutput.write(content.getBytes("UTF-8"));tempOutput.write("\n".getBytes("UTF-8"));// 追加原文件内容FSDataInputStream inputStream = fs.open(hdfsPath);IOUtils.copyBytes(inputStream, tempOutput, conf);inputStream.close();tempOutput.close();// 删除原文件，重命名临时文件fs.delete(hdfsPath, false);fs.rename(tempPath, hdfsPath);System.out.println("内容已追加到文件开头: " + hdfsFilePath);} else {// 追加到结尾FSDataOutputStream outputStream = fs.append(hdfsPath);outputStream.write("\n".getBytes("UTF-8"));outputStream.write(content.getBytes("UTF-8"));outputStream.close();System.out.println("内容已追加到文件结尾: " + hdfsFilePath);}}/*** 删除HDFS中的指定文件* @param hdfsFilePath HDFS文件路径* @throws IOException IO异常*/public void deleteFile(String hdfsFilePath) throws IOException {fileOperation(hdfsFilePath, false);}/*** 在HDFS中将文件从源路径移动到目标路径* @param srcPath 源路径* @param destPath 目标路径* @throws IOException IO异常*/public void moveFile(String srcPath, String destPath) throws IOException {Path source = new Path(srcPath);Path destination = new Path(destPath);// 确保目标路径的父目录存在Path parentPath = destination.getParent();if (parentPath != null && !fs.exists(parentPath)) {fs.mkdirs(parentPath);}boolean success = fs.rename(source, destination);if (success) {System.out.println("文件已从 " + srcPath + " 移动到 " + destPath);} else {System.out.println("移动文件失败");}}/*** 执行HDFS Shell命令的示例方法* 注意：实际使用时需要在命令行中执行这些命令*/public static void showHDFSShellCommands() {System.out.println("\n===== HDFS Shell命令参考 =====");System.out.println("1. 上传文件（追加或覆盖）:");System.out.println(" 上传新文件: hadoop fs -put localfile /hdfs/path");System.out.println(" 覆盖文件: hadoop fs -put -f localfile /hdfs/path");System.out.println(" 追加内容: hadoop fs -appendToFile localfile /hdfs/path");System.out.println("\n2. 下载文件:");System.out.println(" hadoop fs -get /hdfs/path localdir");System.out.println("\n3. 输出文件内容:");System.out.println(" hadoop fs -cat /hdfs/path");System.out.println("\n4. 显示文件信息:");System.out.println(" hadoop fs -ls -h /hdfs/path");System.out.println("\n5. 递归显示目录信息:");System.out.println(" hadoop fs -ls -R /hdfs/dir");System.out.println("\n6. 创建/删除文件:");System.out.println(" 创建空文件: hadoop fs -touchz /hdfs/path");System.out.println(" 删除文件: hadoop fs -rm /hdfs/path");System.out.println("\n7. 创建/删除目录:");System.out.println(" 创建目录: hadoop fs -mkdir -p /hdfs/dir");System.out.println(" 删除空目录: hadoop fs -rmdir /hdfs/dir");System.out.println(" 删除非空目录: hadoop fs -rm -r /hdfs/dir");System.out.println("\n8. 追加内容到文件:");System.out.println(" hadoop fs -appendToFile - /hdfs/path （然后输入内容，按Ctrl+D结束）");System.out.println("\n9. 删除文件:");System.out.println(" hadoop fs -rm /hdfs/path");System.out.println("\n10. 移动文件:");System.out.println(" hadoop fs -mv /hdfs/src /hdfs/dest");}
}Main.java
package com.example.hdfs;import java.io.IOException;
import java.util.Scanner;/*** 主类，用于演示和测试HDFS操作功能*/
public class Main {private static final String DEFAULT_HDFS_URI = "hdfs://192.168.88.151:8020";public static void main(String[] args) {HDFSUtil hdfsUtil = null;try (Scanner scanner = new Scanner(System.in)) {// 初始化HDFS连接System.out.println("正在连接HDFS: " + DEFAULT_HDFS_URI);hdfsUtil = new HDFSUtil(DEFAULT_HDFS_URI);System.out.println("HDFS连接成功！");// 显示菜单showMenu();while (true) {System.out.print("请选择操作 (0-12): ");int choice = scanner.nextInt();scanner.nextLine(); // 消费换行符switch (choice) {case 0:System.out.println("感谢使用，再见！");return;case 1:handleUploadFile(hdfsUtil, scanner);break;case 2:handleDownloadFile(hdfsUtil, scanner);break;case 3:handleCatFile(hdfsUtil, scanner);break;case 4:handleLsFile(hdfsUtil, scanner);break;case 5:handleLsRecursive(hdfsUtil, scanner);break;case 6:handleFileOperation(hdfsUtil, scanner);break;case 7:handleDirOperation(hdfsUtil, scanner);break;case 8:handleAppendToFile(hdfsUtil, scanner);break;case 9:handleDeleteFile(hdfsUtil, scanner);break;case 10:handleMoveFile(hdfsUtil, scanner);break;case 11:HDFSUtil.showHDFSShellCommands();break;case 12:showMenu(); // 重新显示菜单break;default:System.out.println("无效的选择，请重新输入！");}// 仅显示提示，等待用户按Enter键继续System.out.println();scanner.nextLine();}} catch (IOException e) {System.err.println("HDFS操作错误: " + e.getMessage());e.printStackTrace();} finally {// 关闭连接if (hdfsUtil != null) {try {hdfsUtil.close();} catch (IOException e) {System.err.println("关闭HDFS连接时出错: " + e.getMessage());}}}}/*** 显示菜单*/private static void showMenu() {System.out.println("\n======== HDFS操作工具 ========");System.out.println("1. 上传文件（支持追加/覆盖）");System.out.println("2. 下载文件（自动重命名）");System.out.println("3. 查看文件内容");System.out.println("4. 显示文件详细信息");System.out.println("5. 递归显示目录信息");System.out.println("6. 创建/删除文件");System.out.println("7. 创建/删除目录");System.out.println("8. 向文件追加内容（开头/结尾）");System.out.println("9. 删除文件");System.out.println("10. 移动文件");System.out.println("11. 查看HDFS Shell命令参考");System.out.println("12. 重新显示菜单");System.out.println("0. 退出");System.out.println("==============================");}/*** 处理文件上传*/private static void handleUploadFile(HDFSUtil hdfsUtil, Scanner scanner) throws IOException {System.out.print("请输入本地文件路径: ");String localPath = scanner.nextLine();System.out.print("请输入HDFS目标路径: ");String hdfsPath = scanner.nextLine();// 检查文件是否存在if (hdfsUtil.getFileSystem().exists(new org.apache.hadoop.fs.Path(hdfsPath))) {System.out.print("文件已存在，是否追加到文件末尾？(y/n，其他将覆盖): ");String appendChoice = scanner.nextLine();hdfsUtil.uploadFile(localPath, hdfsPath, "y".equalsIgnoreCase(appendChoice));} else {hdfsUtil.uploadFile(localPath, hdfsPath, false);}}/*** 处理文件下载*/private static void handleDownloadFile(HDFSUtil hdfsUtil, Scanner scanner) throws IOException {System.out.print("请输入HDFS文件路径: ");String hdfsPath = scanner.nextLine();System.out.print("请输入本地目录路径: ");String localDir = scanner.nextLine();hdfsUtil.downloadFile(hdfsPath, localDir);}/*** 处理查看文件内容*/private static void handleCatFile(HDFSUtil hdfsUtil, Scanner scanner) throws IOException {System.out.print("请输入HDFS文件路径: ");String hdfsPath = scanner.nextLine();System.out.println("\n文件内容:\n");hdfsUtil.catFile(hdfsPath);System.out.println("\n");}/*** 处理显示文件信息*/private static void handleLsFile(HDFSUtil hdfsUtil, Scanner scanner) throws IOException {System.out.print("请输入HDFS文件路径: ");String hdfsPath = scanner.nextLine();hdfsUtil.lsFile(hdfsPath);}/*** 处理递归显示目录信息*/private static void handleLsRecursive(HDFSUtil hdfsUtil, Scanner scanner) throws IOException {System.out.print("请输入HDFS目录路径: ");String hdfsPath = scanner.nextLine();hdfsUtil.lsRecursive(hdfsPath);}/*** 处理文件创建/删除操作*/private static void handleFileOperation(HDFSUtil hdfsUtil, Scanner scanner) throws IOException {System.out.print("请选择操作 (1-创建文件, 2-删除文件): ");int op = scanner.nextInt();scanner.nextLine(); // 消费换行符System.out.print("请输入HDFS文件路径: ");String hdfsPath = scanner.nextLine();hdfsUtil.fileOperation(hdfsPath, op == 1);}/*** 处理目录创建/删除操作*/private static void handleDirOperation(HDFSUtil hdfsUtil, Scanner scanner) throws IOException {System.out.print("请选择操作 (1-创建目录, 2-删除目录): ");int op = scanner.nextInt();scanner.nextLine(); // 消费换行符System.out.print("请输入HDFS目录路径: ");String hdfsPath = scanner.nextLine();boolean forceDelete = false;if (op == 2) {System.out.print("目录不为空时是否强制删除？(y/n): ");forceDelete = "y".equalsIgnoreCase(scanner.nextLine());}hdfsUtil.dirOperation(hdfsPath, op == 1, forceDelete);}/*** 处理向文件追加内容*/private static void handleAppendToFile(HDFSUtil hdfsUtil, Scanner scanner) throws IOException {System.out.print("请输入HDFS文件路径: ");String hdfsPath = scanner.nextLine();System.out.print("请选择追加位置 (1-开头, 2-结尾): ");int position = scanner.nextInt();scanner.nextLine(); // 消费换行符System.out.print("请输入要追加的内容: ");String content = scanner.nextLine();hdfsUtil.appendToFile(hdfsPath, content, position == 1);}/*** 处理删除文件*/private static void handleDeleteFile(HDFSUtil hdfsUtil, Scanner scanner) throws IOException {System.out.print("请输入HDFS文件路径: ");String hdfsPath = scanner.nextLine();hdfsUtil.deleteFile(hdfsPath);}/*** 处理移动文件*/private static void handleMoveFile(HDFSUtil hdfsUtil, Scanner scanner) throws IOException {System.out.print("请输入源文件路径: ");String srcPath = scanner.nextLine();System.out.print("请输入目标文件路径: ");String destPath = scanner.nextLine();hdfsUtil.moveFile(srcPath, destPath);}
}使用Shell命令实现
1. 上传文件（追加或覆盖）
# 上传新文件
hadoop fs -put localfile /hdfs/path/file.txt# 覆盖已存在文件
hadoop fs -put -f localfile /hdfs/path/file.txt# 追加到已存在文件
hadoop fs -appendToFile localfile /hdfs/path/file.txt2. 下载文件（自动重命名）
# 下载文件，如果本地存在需要手动重命名
hadoop fs -get /hdfs/path/file.txt ./localdir/# 或者使用copyToLocal
hadoop fs -copyToLocal /hdfs/path/file.txt ./localdir/3. 查看文件内容
# 输出文件内容到终端
hadoop fs -cat /hdfs/path/file.txt# 或者使用tail查看文件末尾
hadoop fs -tail /hdfs/path/file.txt4. 显示文件详细信息
# 显示文件详细信息（权限、大小、时间等）
hadoop fs -ls -h /hdfs/path/file.txt# 显示更详细的信息
hadoop fs -ls -d -h /hdfs/path/file.txt5. 递归显示目录信息
# 递归显示目录下所有文件信息
hadoop fs -ls -R -h /hdfs/directory/# 递归显示并包含详细信息
hadoop fs -ls -R -h /hdfs/directory/6. 文件创建和删除
# 创建空文件
hadoop fs -touchz /hdfs/path/newfile.txt# 创建文件并确保目录存在
hadoop fs -mkdir -p /hdfs/parent/directory/
hadoop fs -touchz /hdfs/parent/directory/newfile.txt# 删除文件
hadoop fs -rm /hdfs/path/file.txt# 强制删除文件
hadoop fs -rm -f /hdfs/path/file.txt7. 目录创建和删除
# 创建目录（自动创建父目录）
hadoop fs -mkdir -p /hdfs/new/directory/# 删除空目录
hadoop fs -rmdir /hdfs/empty/directory/# 删除非空目录（递归删除）
hadoop fs -rm -r /hdfs/nonempty/directory/# 强制删除非空目录
hadoop fs -rm -r -f /hdfs/nonempty/directory/8. 向文件追加内容
# 追加内容到文件结尾
echo "new content" | hadoop fs -appendToFile - /hdfs/path/file.txt# 或者从本地文件追加
hadoop fs -appendToFile localfile.txt /hdfs/path/file.txt# 追加到文件开头需要复杂操作（先下载，修改，再上传）
hadoop fs -get /hdfs/path/file.txt ./
echo "header content" | cat - file.txt > temp.txt && mv temp.txt file.txt
hadoop fs -put -f file.txt /hdfs/path/file.txt9. 删除文件
# 删除单个文件
hadoop fs -rm /hdfs/path/file.txt# 删除多个文件
hadoop fs -rm /hdfs/path/file1.txt /hdfs/path/file2.txt# 使用通配符删除
hadoop fs -rm /hdfs/path/*.txt10. 移动/重命名文件
# 移动文件
hadoop fs -mv /hdfs/source/path/file.txt /hdfs/destination/path/file.txt# 重命名文件
hadoop fs -mv /hdfs/oldname.txt /hdfs/newname.txt# 移动目录
hadoop fs -mv /hdfs/source/directory/ /hdfs/destination/directory/
