1/21/2015

From JAVA to HDFS File operations : Read, write, copy, delete, create


From JAVA to HDFS File operations : Read, write, copy, delete, create

1. creating the directory in the HDFS
2. deleting the directory in the HDF
3. copying file from local to HDFS
4. Read File From HDFS
5. Write File To HDFS

 Change below configuration to your .xml files and use it. 

 
Configuration conf = new Configuration();
conf.addResource(new Path("C:/hadoop-2.5.1/etc/hadoop/core-site.xml"));
conf.addResource(new Path("C:/hadoop-2.5.1/etc/hadoop/hdfs-site.xml"));

-----------------------------------------------------------------------










  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107


package com.my.cert.example;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class HDFSFileOperations {

 public static void main(String args[]) throws IOException {
  Configuration conf = new Configuration();
  conf.addResource(new Path("C:/hadoop-2.5.1/etc/hadoop/core-site.xml"));
  conf.addResource(new Path("C:/hadoop-2.5.1/etc/hadoop/hdfs-site.xml"));
  FileSystem hdfs = FileSystem.get(conf);
  System.out.println("Home Dir: " + getHomedirectory(hdfs));
  System.out.println("Create Directory : "+ createDirectory(hdfs, "/testFolder"));
  System.out.println("copy File From Local: "+ copyFileFromLocal(hdfs, "testFolder","C:/hadoop/test/test.txt"));
  readDataFromHDFSFile(hdfs, "testFolder/test.txt");
  writingDataToHDFS(hdfs, "testFolder/test.txt");

 }

 public static Path getHomedirectory(FileSystem hdfc) throws IOException {
  Path homeDir = hdfc.getHomeDirectory();

  return homeDir;
 }

 /*
  * creating the directory in the HDFS
  */

 public static boolean createDirectory(FileSystem hdfs, String dirName)
   throws IOException {
  Path homeDir = getHomedirectory(hdfs);
  Path newFolderPath = new Path(dirName);
  newFolderPath = Path.mergePaths(homeDir, newFolderPath);
  if (hdfs.exists(newFolderPath)) {
   hdfs.delete(newFolderPath, true);

  }

  return hdfs.mkdirs(newFolderPath);
 }

 /*
  * deleting the directory in the HDFS
  */
 public static boolean deleteDirectory(FileSystem hdfs, String dirName)
   throws IOException {
  Path deleteFolderName = new Path(dirName);
  if (hdfs.exists(deleteFolderName)) {
   return hdfs.delete(deleteFolderName, true);
  }
  return false;
 }

 /*
  * copying file from local to HDFS
  */

 public static boolean copyFileFromLocal(FileSystem hdfs,
   String hdfsFolderName, String localFileAbsPath) throws IOException {
  Path localFilePath = new Path(localFileAbsPath);
  String localFileName = new File(localFileAbsPath).getName();
  Path hdfsFolderpath = new Path(hdfsFolderName + "/" + localFileName);

  if (!hdfs.exists(hdfsFolderpath)) {
   hdfs.createNewFile(hdfsFolderpath);
  }

  hdfs.copyFromLocalFile(localFilePath, hdfsFolderpath);
  return true;
 }

 public static void readDataFromHDFSFile(FileSystem hdfs, String filePath)
   throws IllegalArgumentException, IOException {
  BufferedReader bfr = new BufferedReader(new InputStreamReader(
    hdfs.open(new Path(filePath))));
  String str = null;
  while ((str = bfr.readLine()) != null) {
   System.out.println(str);

  }

 }

 public static void writingDataToHDFS(FileSystem hdfs, String filePath)
   throws IllegalArgumentException, IOException {
  StringBuilder sb = new StringBuilder();
  for (int i = 1; i <= 5; i++) {
   sb.append("Test creating file" + i);
   sb.append("\n");
  }
  byte[] byt = sb.toString().getBytes();
  FSDataOutputStream fsOutStream = hdfs.create(new Path(filePath));
  fsOutStream.write(byt);

  fsOutStream.close();
 }

}



















Posted by



at






















Labels:


















2 comments:































        

      









Subscribe to:
Post Comments (Atom)