Patch for NDFS's df.java

Previous Topic Next Topic
 
classic Classic list List threaded Threaded
3 messages Options
Reply | Threaded
Open this post in threaded view
|

Patch for NDFS's df.java

Dominik Friedrich
This is a patch for NDFS's df.java. With this patch I was able to run
NDFS datanode on Windows systems without cygwin. I haven't found a way
to get the partition size on Windows so it's always set to two times the
available space. Maybe somebody can clean this up so it can be included
into nutch trunk.

best regards,
Dominik

Index: DF.java
===================================================================
--- DF.java (revision 370204)
+++ DF.java (working copy)
@@ -15,81 +15,167 @@
  */
 package org.apache.nutch.ndfs;
 
-import java.io.File;
+import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
-import java.io.BufferedReader;
-
 import java.util.StringTokenizer;
-import java.util.Iterator;
 
-/** Filesystem disk space usage statistics.  Uses the unix 'df' program.
- * Tested on Linux, FreeBSD and Cygwin. */
+/**
+ * Filesystem disk space usage statistics. Uses the unix 'df' program. Tested on Linux, FreeBSD and Cygwin.
+ */
 public class DF {
-  private String filesystem;
-  private long capacity;
-  private long used;
-  private long available;
-  private int percentUsed;
-  private String mount;
-  
-  public DF(String path) throws IOException {
+ private String filesystem;
 
-    Process process = Runtime.getRuntime().exec(new String[] {"df","-k",path});
+ private long capacity;
 
-    try {
-      if (process.waitFor() == 0) {
-        BufferedReader lines =
-          new BufferedReader(new InputStreamReader(process.getInputStream()));
+ private long used;
 
-        lines.readLine();                         // skip headings
+ private long available;
 
-        StringTokenizer tokens =
-          new StringTokenizer(lines.readLine(), " \t\n\r\f%");
-        
-        this.filesystem = tokens.nextToken();
-        if (!tokens.hasMoreTokens()) {            // for long filesystem name
-          tokens = new StringTokenizer(lines.readLine(), " \t\n\r\f%");
-        }
-        this.capacity = Long.parseLong(tokens.nextToken()) * 1024;
-        this.used = Long.parseLong(tokens.nextToken()) * 1024;
-        this.available = Long.parseLong(tokens.nextToken()) * 1024;
-        this.percentUsed = Integer.parseInt(tokens.nextToken());
-        this.mount = tokens.nextToken();
+ private int percentUsed;
 
-      } else {
-        throw new IOException
-          (new BufferedReader(new InputStreamReader(process.getErrorStream()))
-           .readLine());
-      }
-    } catch (InterruptedException e) {
-      throw new IOException(e.toString());
-    } finally {
-      process.destroy();
-    }
-  }
+ private String mount;
 
-  /// ACCESSORS
+ public DF(String path) throws IOException {
+ String os = System.getProperty("os.name");
+ if(os.startsWith("Windows"))
+ dfWidows(path);
+ else
+ dfUnix(path);
+ }
 
-  public String getFilesystem() { return filesystem; }
-  public long getCapacity() { return capacity; }
-  public long getUsed() { return used; }
-  public long getAvailable() { return available; }
-  public int getPercentUsed() { return percentUsed; }
-  public String getMount() { return mount; }
-  
-  public String toString() {
-    return
-      "df -k " + mount +"\n" +
-      filesystem + "\t" +
-      capacity / 1024 + "\t" +
-      used / 1024 + "\t" +
-      available / 1024 + "\t" +
-      percentUsed + "%\t" +
-      mount;
-  }
+ // / ACCESSORS
 
-  public static void main(String[] args) throws Exception {
-    System.out.println(new DF(args[0]));
-  }
+ /**
+ * @return  Returns the filesystem.
+ * @uml.property  name="filesystem"
+ */
+ public String getFilesystem() {
+ return filesystem;
+ }
+
+ /**
+ * @return  Returns the capacity.
+ * @uml.property  name="capacity"
+ */
+ public long getCapacity() {
+ return capacity;
+ }
+
+ /**
+ * @return  Returns the used.
+ * @uml.property  name="used"
+ */
+ public long getUsed() {
+ return used;
+ }
+
+ /**
+ * @return  Returns the available.
+ * @uml.property  name="available"
+ */
+ public long getAvailable() {
+ return available;
+ }
+
+ /**
+ * @return  Returns the percentUsed.
+ * @uml.property  name="percentUsed"
+ */
+ public int getPercentUsed() {
+ return percentUsed;
+ }
+
+ /**
+ * @return  Returns the mount.
+ * @uml.property  name="mount"
+ */
+ public String getMount() {
+ return mount;
+ }
+
+ public String toString() {
+ return "df -k " + mount + "\n" + filesystem + "\t" + capacity / 1024
+ + "\t" + used / 1024 + "\t" + available / 1024 + "\t"
+ + percentUsed + "%\t" + mount;
+ }
+
+ private void dfUnix(String path) throws IOException {
+ Process process = Runtime.getRuntime().exec(
+ new String[] { "df", "-k", path });
+
+ try {
+ if (process.waitFor() == 0) {
+ BufferedReader lines = new BufferedReader(
+ new InputStreamReader(process.getInputStream()));
+
+ lines.readLine(); // skip headings
+
+ StringTokenizer tokens = new StringTokenizer(lines.readLine(),
+ " \t\n\r\f%");
+
+ this.filesystem = tokens.nextToken();
+ if (!tokens.hasMoreTokens()) { // for long filesystem name
+ tokens = new StringTokenizer(lines.readLine(), " \t\n\r\f%");
+ }
+ this.capacity = Long.parseLong(tokens.nextToken()) * 1024;
+ this.used = Long.parseLong(tokens.nextToken()) * 1024;
+ this.available = Long.parseLong(tokens.nextToken()) * 1024;
+ this.percentUsed = Integer.parseInt(tokens.nextToken());
+ this.mount = tokens.nextToken();
+
+ } else {
+ throw new IOException(new BufferedReader(new InputStreamReader(
+ process.getErrorStream())).readLine());
+ }
+ } catch (InterruptedException e) {
+ throw new IOException(e.toString());
+ } finally {
+ process.destroy();
+ }
+
+ }
+
+ private void dfWidows(String path) throws IOException {
+ try {
+ String os = System.getProperty("os.name");
+ String command;
+ if (os.equals("Windows NT") || os.equals("Windows 2000")) {
+ command = "cmd.exe /c dir " + path;
+ } else {
+ command = "command.com /c dir " + path;
+ }
+ Runtime runtime = Runtime.getRuntime();
+ Process process = runtime.exec(command);
+
+ BufferedReader in = new BufferedReader(new InputStreamReader(
+ process.getInputStream()));
+ String line;
+ String freeSpace = null;
+ while ((line = in.readLine()) != null) {
+ freeSpace = line;
+ }
+ process.destroy();
+ freeSpace = freeSpace.trim();
+ freeSpace = freeSpace.replaceAll("\\.", "");
+ freeSpace = freeSpace.replaceAll(",", "");
+ String[] items = freeSpace.split(" ");
+ int index = 1;
+ while (index < items.length) {
+ try {
+ available = Long.parseLong(items[index++]);
+ capacity = 2*available;
+ used = available;
+ percentUsed = 50;
+ mount = path.substring(0,2);
+ } catch (NumberFormatException nfe) {
+ }
+ }
+ } catch (Exception exception) {
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ System.out.println(new DF(args[0]));
+ }
 }
Reply | Threaded
Open this post in threaded view
|

Re: Patch for NDFS's df.java

Dominik Friedrich
This is a bugfixed version. I wasn't running a normal NDFS datanode but
a jmx based version I'm working on and I didn't recorgnize that the
first version always returned 0 when the NDFS datanode was started with
no data dir.

best regards,
Dominik

Dominik Friedrich schrieb:
> This is a patch for NDFS's df.java. With this patch I was able to run
> NDFS datanode on Windows systems without cygwin. I haven't found a way
> to get the partition size on Windows so it's always set to two times
> the available space. Maybe somebody can clean this up so it can be
> included into nutch trunk.
>
> best regards,
> Dominik


Index: DF.java
===================================================================
--- DF.java (revision 370204)
+++ DF.java (working copy)
@@ -15,81 +15,167 @@
  */
 package org.apache.nutch.ndfs;
 
-import java.io.File;
+import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;
-import java.io.BufferedReader;
-
 import java.util.StringTokenizer;
-import java.util.Iterator;
 
-/** Filesystem disk space usage statistics.  Uses the unix 'df' program.
- * Tested on Linux, FreeBSD and Cygwin. */
+/**
+ * Filesystem disk space usage statistics. Uses the unix 'df' program. Tested on Linux, FreeBSD and Cygwin.
+ */
 public class DF {
-  private String filesystem;
-  private long capacity;
-  private long used;
-  private long available;
-  private int percentUsed;
-  private String mount;
-  
-  public DF(String path) throws IOException {
+ private String filesystem;
 
-    Process process = Runtime.getRuntime().exec(new String[] {"df","-k",path});
+ private long capacity;
 
-    try {
-      if (process.waitFor() == 0) {
-        BufferedReader lines =
-          new BufferedReader(new InputStreamReader(process.getInputStream()));
+ private long used;
 
-        lines.readLine();                         // skip headings
+ private long available;
 
-        StringTokenizer tokens =
-          new StringTokenizer(lines.readLine(), " \t\n\r\f%");
-        
-        this.filesystem = tokens.nextToken();
-        if (!tokens.hasMoreTokens()) {            // for long filesystem name
-          tokens = new StringTokenizer(lines.readLine(), " \t\n\r\f%");
-        }
-        this.capacity = Long.parseLong(tokens.nextToken()) * 1024;
-        this.used = Long.parseLong(tokens.nextToken()) * 1024;
-        this.available = Long.parseLong(tokens.nextToken()) * 1024;
-        this.percentUsed = Integer.parseInt(tokens.nextToken());
-        this.mount = tokens.nextToken();
+ private int percentUsed;
 
-      } else {
-        throw new IOException
-          (new BufferedReader(new InputStreamReader(process.getErrorStream()))
-           .readLine());
-      }
-    } catch (InterruptedException e) {
-      throw new IOException(e.toString());
-    } finally {
-      process.destroy();
-    }
-  }
+ private String mount;
 
-  /// ACCESSORS
+ public DF(String path) throws IOException {
+ String os = System.getProperty("os.name");
+ if(os.startsWith("Windows"))
+ dfWidows(path);
+ else
+ dfUnix(path);
+ }
 
-  public String getFilesystem() { return filesystem; }
-  public long getCapacity() { return capacity; }
-  public long getUsed() { return used; }
-  public long getAvailable() { return available; }
-  public int getPercentUsed() { return percentUsed; }
-  public String getMount() { return mount; }
-  
-  public String toString() {
-    return
-      "df -k " + mount +"\n" +
-      filesystem + "\t" +
-      capacity / 1024 + "\t" +
-      used / 1024 + "\t" +
-      available / 1024 + "\t" +
-      percentUsed + "%\t" +
-      mount;
-  }
+ // / ACCESSORS
 
-  public static void main(String[] args) throws Exception {
-    System.out.println(new DF(args[0]));
-  }
+ /**
+ * @return  Returns the filesystem.
+ * @uml.property  name="filesystem"
+ */
+ public String getFilesystem() {
+ return filesystem;
+ }
+
+ /**
+ * @return  Returns the capacity.
+ * @uml.property  name="capacity"
+ */
+ public long getCapacity() {
+ return capacity;
+ }
+
+ /**
+ * @return  Returns the used.
+ * @uml.property  name="used"
+ */
+ public long getUsed() {
+ return used;
+ }
+
+ /**
+ * @return  Returns the available.
+ * @uml.property  name="available"
+ */
+ public long getAvailable() {
+ return available;
+ }
+
+ /**
+ * @return  Returns the percentUsed.
+ * @uml.property  name="percentUsed"
+ */
+ public int getPercentUsed() {
+ return percentUsed;
+ }
+
+ /**
+ * @return  Returns the mount.
+ * @uml.property  name="mount"
+ */
+ public String getMount() {
+ return mount;
+ }
+
+ public String toString() {
+ return "df -k " + mount + "\n" + filesystem + "\t" + capacity / 1024
+ + "\t" + used / 1024 + "\t" + available / 1024 + "\t"
+ + percentUsed + "%\t" + mount;
+ }
+
+ private void dfUnix(String path) throws IOException {
+ Process process = Runtime.getRuntime().exec(
+ new String[] { "df", "-k", path });
+
+ try {
+ if (process.waitFor() == 0) {
+ BufferedReader lines = new BufferedReader(
+ new InputStreamReader(process.getInputStream()));
+
+ lines.readLine(); // skip headings
+
+ StringTokenizer tokens = new StringTokenizer(lines.readLine(),
+ " \t\n\r\f%");
+
+ this.filesystem = tokens.nextToken();
+ if (!tokens.hasMoreTokens()) { // for long filesystem name
+ tokens = new StringTokenizer(lines.readLine(), " \t\n\r\f%");
+ }
+ this.capacity = Long.parseLong(tokens.nextToken()) * 1024;
+ this.used = Long.parseLong(tokens.nextToken()) * 1024;
+ this.available = Long.parseLong(tokens.nextToken()) * 1024;
+ this.percentUsed = Integer.parseInt(tokens.nextToken());
+ this.mount = tokens.nextToken();
+
+ } else {
+ throw new IOException(new BufferedReader(new InputStreamReader(
+ process.getErrorStream())).readLine());
+ }
+ } catch (InterruptedException e) {
+ throw new IOException(e.toString());
+ } finally {
+ process.destroy();
+ }
+
+ }
+
+ private void dfWidows(String path) throws IOException {
+ try {
+ String os = System.getProperty("os.name");
+ String command;
+ if (os.equals("Windows NT") || os.equals("Windows 2000")) {
+ command = "cmd.exe /c dir " + path.substring(0, 3);
+ } else {
+ command = "command.com /c dir " + path.substring(0, 3);
+ }
+ Runtime runtime = Runtime.getRuntime();
+ Process process = runtime.exec(command);
+
+ BufferedReader in = new BufferedReader(new InputStreamReader(
+ process.getInputStream()));
+ String line;
+ String freeSpace = null;
+ while ((line = in.readLine()) != null) {
+ freeSpace = line;
+ }
+ process.destroy();
+ freeSpace = freeSpace.trim();
+ freeSpace = freeSpace.replaceAll("\\.", "");
+ freeSpace = freeSpace.replaceAll(",", "");
+ String[] items = freeSpace.split(" ");
+ int index = 1;
+ while (index < items.length) {
+ try {
+ available = Long.parseLong(items[index++]);
+ capacity = 2*available;
+ used = available;
+ percentUsed = 50;
+ mount = path.substring(0,2);
+ } catch (NumberFormatException nfe) {
+ }
+ }
+ } catch (Exception exception) {
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ System.out.println(new DF(args[0]));
+ }
 }
Reply | Threaded
Open this post in threaded view
|

Re: Patch for NDFS's df.java

Stefan Groschupf-2
+1
May you can clean up thinks by your self and create a jira issue  
where you can attach this patch.
Thanks.
Stefan

Am 20.01.2006 um 16:33 schrieb Dominik Friedrich:

> This is a bugfixed version. I wasn't running a normal NDFS datanode  
> but a jmx based version I'm working on and I didn't recorgnize that  
> the first version always returned 0 when the NDFS datanode was  
> started with no data dir.
>
> best regards,
> Dominik
>
> Dominik Friedrich schrieb:
>> This is a patch for NDFS's df.java. With this patch I was able to  
>> run NDFS datanode on Windows systems without cygwin. I haven't  
>> found a way to get the partition size on Windows so it's always  
>> set to two times the available space. Maybe somebody can clean  
>> this up so it can be included into nutch trunk.
>>
>> best regards,
>> Dominik
>
> Index: DF.java
> ===================================================================
> --- DF.java (revision 370204)
> +++ DF.java (working copy)
> @@ -15,81 +15,167 @@
>   */
>  package org.apache.nutch.ndfs;
>
> -import java.io.File;
> +import java.io.BufferedReader;
>  import java.io.IOException;
>  import java.io.InputStreamReader;
> -import java.io.BufferedReader;
> -
>  import java.util.StringTokenizer;
> -import java.util.Iterator;
>
> -/** Filesystem disk space usage statistics.  Uses the unix 'df'  
> program.
> - * Tested on Linux, FreeBSD and Cygwin. */
> +/**
> + * Filesystem disk space usage statistics. Uses the unix 'df'  
> program. Tested on Linux, FreeBSD and Cygwin.
> + */
>  public class DF {
> -  private String filesystem;
> -  private long capacity;
> -  private long used;
> -  private long available;
> -  private int percentUsed;
> -  private String mount;
> -
> -  public DF(String path) throws IOException {
> + private String filesystem;
>
> -    Process process = Runtime.getRuntime().exec(new String[]  
> {"df","-k",path});
> + private long capacity;
>
> -    try {
> -      if (process.waitFor() == 0) {
> -        BufferedReader lines =
> -          new BufferedReader(new InputStreamReader
> (process.getInputStream()));
> + private long used;
>
> -        lines.readLine();                         // skip headings
> + private long available;
>
> -        StringTokenizer tokens =
> -          new StringTokenizer(lines.readLine(), " \t\n\r\f%");
> -
> -        this.filesystem = tokens.nextToken();
> -        if (!tokens.hasMoreTokens()) {            // for long  
> filesystem name
> -          tokens = new StringTokenizer(lines.readLine(), " \t\n\r\f
> %");
> -        }
> -        this.capacity = Long.parseLong(tokens.nextToken()) * 1024;
> -        this.used = Long.parseLong(tokens.nextToken()) * 1024;
> -        this.available = Long.parseLong(tokens.nextToken()) * 1024;
> -        this.percentUsed = Integer.parseInt(tokens.nextToken());
> -        this.mount = tokens.nextToken();
> + private int percentUsed;
>
> -      } else {
> -        throw new IOException
> -          (new BufferedReader(new InputStreamReader
> (process.getErrorStream()))
> -           .readLine());
> -      }
> -    } catch (InterruptedException e) {
> -      throw new IOException(e.toString());
> -    } finally {
> -      process.destroy();
> -    }
> -  }
> + private String mount;
>
> -  /// ACCESSORS
> + public DF(String path) throws IOException {
> + String os = System.getProperty("os.name");
> + if(os.startsWith("Windows"))
> + dfWidows(path);
> + else
> + dfUnix(path);
> + }
>
> -  public String getFilesystem() { return filesystem; }
> -  public long getCapacity() { return capacity; }
> -  public long getUsed() { return used; }
> -  public long getAvailable() { return available; }
> -  public int getPercentUsed() { return percentUsed; }
> -  public String getMount() { return mount; }
> -
> -  public String toString() {
> -    return
> -      "df -k " + mount +"\n" +
> -      filesystem + "\t" +
> -      capacity / 1024 + "\t" +
> -      used / 1024 + "\t" +
> -      available / 1024 + "\t" +
> -      percentUsed + "%\t" +
> -      mount;
> -  }
> + // / ACCESSORS
>
> -  public static void main(String[] args) throws Exception {
> -    System.out.println(new DF(args[0]));
> -  }
> + /**
> + * @return  Returns the filesystem.
> + * @uml.property  name="filesystem"
> + */
> + public String getFilesystem() {
> + return filesystem;
> + }
> +
> + /**
> + * @return  Returns the capacity.
> + * @uml.property  name="capacity"
> + */
> + public long getCapacity() {
> + return capacity;
> + }
> +
> + /**
> + * @return  Returns the used.
> + * @uml.property  name="used"
> + */
> + public long getUsed() {
> + return used;
> + }
> +
> + /**
> + * @return  Returns the available.
> + * @uml.property  name="available"
> + */
> + public long getAvailable() {
> + return available;
> + }
> +
> + /**
> + * @return  Returns the percentUsed.
> + * @uml.property  name="percentUsed"
> + */
> + public int getPercentUsed() {
> + return percentUsed;
> + }
> +
> + /**
> + * @return  Returns the mount.
> + * @uml.property  name="mount"
> + */
> + public String getMount() {
> + return mount;
> + }
> +
> + public String toString() {
> + return "df -k " + mount + "\n" + filesystem + "\t" + capacity /  
> 1024
> + + "\t" + used / 1024 + "\t" + available / 1024 + "\t"
> + + percentUsed + "%\t" + mount;
> + }
> +
> + private void dfUnix(String path) throws IOException {
> + Process process = Runtime.getRuntime().exec(
> + new String[] { "df", "-k", path });
> +
> + try {
> + if (process.waitFor() == 0) {
> + BufferedReader lines = new BufferedReader(
> + new InputStreamReader(process.getInputStream()));
> +
> + lines.readLine(); // skip headings
> +
> + StringTokenizer tokens = new StringTokenizer(lines.readLine(),
> + " \t\n\r\f%");
> +
> + this.filesystem = tokens.nextToken();
> + if (!tokens.hasMoreTokens()) { // for long filesystem name
> + tokens = new StringTokenizer(lines.readLine(), " \t\n\r\f%");
> + }
> + this.capacity = Long.parseLong(tokens.nextToken()) * 1024;
> + this.used = Long.parseLong(tokens.nextToken()) * 1024;
> + this.available = Long.parseLong(tokens.nextToken()) * 1024;
> + this.percentUsed = Integer.parseInt(tokens.nextToken());
> + this.mount = tokens.nextToken();
> +
> + } else {
> + throw new IOException(new BufferedReader(new InputStreamReader(
> + process.getErrorStream())).readLine());
> + }
> + } catch (InterruptedException e) {
> + throw new IOException(e.toString());
> + } finally {
> + process.destroy();
> + }
> +
> + }
> +
> + private void dfWidows(String path) throws IOException {
> + try {
> + String os = System.getProperty("os.name");
> + String command;
> + if (os.equals("Windows NT") || os.equals("Windows 2000")) {
> + command = "cmd.exe /c dir " + path.substring(0, 3);
> + } else {
> + command = "command.com /c dir " + path.substring(0, 3);
> + }
> + Runtime runtime = Runtime.getRuntime();
> + Process process = runtime.exec(command);
> +
> + BufferedReader in = new BufferedReader(new InputStreamReader(
> + process.getInputStream()));
> + String line;
> + String freeSpace = null;
> + while ((line = in.readLine()) != null) {
> + freeSpace = line;
> + }
> + process.destroy();
> + freeSpace = freeSpace.trim();
> + freeSpace = freeSpace.replaceAll("\\.", "");
> + freeSpace = freeSpace.replaceAll(",", "");
> + String[] items = freeSpace.split(" ");
> + int index = 1;
> + while (index < items.length) {
> + try {
> + available = Long.parseLong(items[index++]);
> + capacity = 2*available;
> + used = available;
> + percentUsed = 50;
> + mount = path.substring(0,2);
> + } catch (NumberFormatException nfe) {
> + }
> + }
> + } catch (Exception exception) {
> + }
> + }
> +
> + public static void main(String[] args) throws Exception {
> + System.out.println(new DF(args[0]));
> + }
>  }

---------------------------------------------------------------
company:        http://www.media-style.com
forum:        http://www.text-mining.org
blog:            http://www.find23.net