Patch to remove Nutch formating from logs

classic Classic list List threaded Threaded
3 messages Options
Reply | Threaded
Open this post in threaded view
|

Patch to remove Nutch formating from logs

Christopher Burkey
Hello,

    Here is a patch to change org.apache.nutch.util.LogFormatter to not
insert itself as the default handler for the system.

    I have been using Nutch for a year and have been waiting for a
version that I can embed into OpenEdit. The problem has been that Nutch
inserts itself as the formatter for the Java log system and that
interferes with OpenEdit logging.


--
513-542-3401
[hidden email]
http://www.openedit.org


diff -Naur ../java/org/apache/nutch/util/LogFormatter.java java/org/apache/nutch/util/LogFormatter.java
--- ../java/org/apache/nutch/util/LogFormatter.java 2006-03-31 13:40:50.000000000 -0500
+++ java/org/apache/nutch/util/LogFormatter.java 2006-04-05 16:27:59.000000000 -0400
@@ -16,13 +16,23 @@
 
 package org.apache.nutch.util;
 
-import java.util.logging.*;
-import java.io.*;
-import java.text.*;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.text.FieldPosition;
+import java.text.SimpleDateFormat;
 import java.util.Date;
-
-/** Prints just the date and the log message. */
-
+import java.util.logging.Formatter;
+import java.util.logging.Level;
+import java.util.logging.LogRecord;
+import java.util.logging.Logger;
+
+/** Prints just the date and the log message.
+ *  This was also used to stop processing as nutch crawls a web site
+ *  [hidden email] changed this code to use a LogWrapper class to catch severe errors
+ * */
 public class LogFormatter extends Formatter {
   private static final String FORMAT = "yyMMdd HHmmss";
   private static final String NEWLINE = System.getProperty("line.separator");
@@ -35,20 +45,27 @@
   private static boolean showTime = true;
   private static boolean showThreadIDs = false;
 
+  protected static LogFormatter sharedformatter =  new LogFormatter();
+  protected static SevereLogHandler sharedhandler =  new SevereLogHandler(sharedformatter);
+
+  /*
   // install when this class is loaded
   static {
     Handler[] handlers = LogFormatter.getLogger("").getHandlers();
     for (int i = 0; i < handlers.length; i++) {
-      handlers[i].setFormatter(new LogFormatter());
+      handlers[i].setFormatter(sharedformatter);
       handlers[i].setLevel(Level.FINEST);
     }
   }
-
+  */
   /** Gets a logger and, as a side effect, installs this as the default
    * formatter. */
   public static Logger getLogger(String name) {
     // just referencing this class installs it
-    return Logger.getLogger(name);
+ Logger logr = Logger.getLogger(name);
+ logr.addHandler(sharedhandler);
+
+ return logr;
   }
   
   /** When true, time is logged with each entry. */
@@ -60,7 +77,10 @@
   public static void setShowThreadIDs(boolean showThreadIDs) {
     LogFormatter.showThreadIDs = showThreadIDs;
   }
-
+  public void setLoggedSevere( boolean inSevere )
+  {
+  loggedSevere = inSevere;
+  }
   /**
    * Format the given LogRecord.
    * @param record the log record to be formatted.
diff -Naur ../java/org/apache/nutch/util/SevereLogHandler.java java/org/apache/nutch/util/SevereLogHandler.java
--- ../java/org/apache/nutch/util/SevereLogHandler.java 1969-12-31 19:00:00.000000000 -0500
+++ java/org/apache/nutch/util/SevereLogHandler.java 2006-04-05 16:29:20.000000000 -0400
@@ -0,0 +1,46 @@
+/*
+ * Created on Apr 5, 2006
+ */
+package org.apache.nutch.util;
+
+import java.util.logging.Handler;
+import java.util.logging.Level;
+import java.util.logging.LogRecord;
+
+public class SevereLogHandler extends Handler
+{
+ protected LogFormatter fieldNutchFormatter;
+
+ public SevereLogHandler(LogFormatter inFormatter)
+ {
+ setNutchFormatter(inFormatter);
+ }
+
+ protected LogFormatter getNutchFormatter()
+ {
+ return fieldNutchFormatter;
+ }
+
+ protected void setNutchFormatter(LogFormatter inNutchFormatter)
+ {
+ fieldNutchFormatter = inNutchFormatter;
+ }
+
+ public void publish(LogRecord inRecord)
+ {
+ if ( inRecord.getLevel().intValue() == Level.SEVERE.intValue())
+ {
+ getNutchFormatter().setLoggedSevere(true);
+ }
+ }
+
+ public void flush()
+ {
+ }
+
+ public void close() throws SecurityException
+ {
+ }
+
+
+}
Reply | Threaded
Open this post in threaded view
|

Re: Patch to remove Nutch formating from logs

Christopher Burkey
Did anyone get this email? Can a commiter acknowledge this has been
received?

We are have been having problems with infinite loops caused by Nutch. My
theory is that the problem is related to using the log API to track
severe errors.  This patch is a only a few lines of code and should be
easy to insert. Please let me know if it has been received and what the
feedback is.



Christopher Burkey wrote:

> Hello,
>
>    Here is a patch to change org.apache.nutch.util.LogFormatter to not
> insert itself as the default handler for the system.
>
>    I have been using Nutch for a year and have been waiting for a
> version that I can embed into OpenEdit. The problem has been that
> Nutch inserts itself as the formatter for the Java log system and that
> interferes with OpenEdit logging.
>
>
> ------------------------------------------------------------------------
>
> diff -Naur ../java/org/apache/nutch/util/LogFormatter.java java/org/apache/nutch/util/LogFormatter.java
> --- ../java/org/apache/nutch/util/LogFormatter.java 2006-03-31 13:40:50.000000000 -0500
> +++ java/org/apache/nutch/util/LogFormatter.java 2006-04-05 16:27:59.000000000 -0400
> @@ -16,13 +16,23 @@
>  
>  package org.apache.nutch.util;
>  
> -import java.util.logging.*;
> -import java.io.*;
> -import java.text.*;
> +import java.io.ByteArrayOutputStream;
> +import java.io.IOException;
> +import java.io.PrintStream;
> +import java.io.PrintWriter;
> +import java.io.StringWriter;
> +import java.text.FieldPosition;
> +import java.text.SimpleDateFormat;
>  import java.util.Date;
> -
> -/** Prints just the date and the log message. */
> -
> +import java.util.logging.Formatter;
> +import java.util.logging.Level;
> +import java.util.logging.LogRecord;
> +import java.util.logging.Logger;
> +
> +/** Prints just the date and the log message.
> + *  This was also used to stop processing as nutch crawls a web site
> + *  [hidden email] changed this code to use a LogWrapper class to catch severe errors
> + * */
>  public class LogFormatter extends Formatter {
>    private static final String FORMAT = "yyMMdd HHmmss";
>    private static final String NEWLINE = System.getProperty("line.separator");
> @@ -35,20 +45,27 @@
>    private static boolean showTime = true;
>    private static boolean showThreadIDs = false;
>  
> +  protected static LogFormatter sharedformatter =  new LogFormatter();
> +  protected static SevereLogHandler sharedhandler =  new SevereLogHandler(sharedformatter);
> +
> +  /*
>    // install when this class is loaded
>    static {
>      Handler[] handlers = LogFormatter.getLogger("").getHandlers();
>      for (int i = 0; i < handlers.length; i++) {
> -      handlers[i].setFormatter(new LogFormatter());
> +      handlers[i].setFormatter(sharedformatter);
>        handlers[i].setLevel(Level.FINEST);
>      }
>    }
> -
> +  */
>    /** Gets a logger and, as a side effect, installs this as the default
>     * formatter. */
>    public static Logger getLogger(String name) {
>      // just referencing this class installs it
> -    return Logger.getLogger(name);
> + Logger logr = Logger.getLogger(name);
> + logr.addHandler(sharedhandler);
> +
> + return logr;
>    }
>    
>    /** When true, time is logged with each entry. */
> @@ -60,7 +77,10 @@
>    public static void setShowThreadIDs(boolean showThreadIDs) {
>      LogFormatter.showThreadIDs = showThreadIDs;
>    }
> -
> +  public void setLoggedSevere( boolean inSevere )
> +  {
> +  loggedSevere = inSevere;
> +  }
>    /**
>     * Format the given LogRecord.
>     * @param record the log record to be formatted.
> diff -Naur ../java/org/apache/nutch/util/SevereLogHandler.java java/org/apache/nutch/util/SevereLogHandler.java
> --- ../java/org/apache/nutch/util/SevereLogHandler.java 1969-12-31 19:00:00.000000000 -0500
> +++ java/org/apache/nutch/util/SevereLogHandler.java 2006-04-05 16:29:20.000000000 -0400
> @@ -0,0 +1,46 @@
> +/*
> + * Created on Apr 5, 2006
> + */
> +package org.apache.nutch.util;
> +
> +import java.util.logging.Handler;
> +import java.util.logging.Level;
> +import java.util.logging.LogRecord;
> +
> +public class SevereLogHandler extends Handler
> +{
> + protected LogFormatter fieldNutchFormatter;
> +
> + public SevereLogHandler(LogFormatter inFormatter)
> + {
> + setNutchFormatter(inFormatter);
> + }
> +
> + protected LogFormatter getNutchFormatter()
> + {
> + return fieldNutchFormatter;
> + }
> +
> + protected void setNutchFormatter(LogFormatter inNutchFormatter)
> + {
> + fieldNutchFormatter = inNutchFormatter;
> + }
> +
> + public void publish(LogRecord inRecord)
> + {
> + if ( inRecord.getLevel().intValue() == Level.SEVERE.intValue())
> + {
> + getNutchFormatter().setLoggedSevere(true);
> + }
> + }
> +
> + public void flush()
> + {
> + }
> +
> + public void close() throws SecurityException
> + {
> + }
> +
> +
> +}
>  


--
513-542-3401
[hidden email]
http://www.openedit.org

Reply | Threaded
Open this post in threaded view
|

Re: Patch to remove Nutch formating from logs

Piotr Kosiorowski
Hello  Christopher,
I personally do not like combining logging with severe error handling
but it is one of the features of Nutch for some time and I do not think
it causes infinite loops in normal installations. Changing it as we are
preparing to release a new version is not a good idea in my opinion.
But I will be happy if we change the way it is handled in future.
So for now -1.
Piotr


Christopher Burkey wrote:

> Did anyone get this email? Can a commiter acknowledge this has been
> received?
>
> We are have been having problems with infinite loops caused by Nutch. My
> theory is that the problem is related to using the log API to track
> severe errors.  This patch is a only a few lines of code and should be
> easy to insert. Please let me know if it has been received and what the
> feedback is.
>
>
>
> Christopher Burkey wrote:
>> Hello,
>>
>>    Here is a patch to change org.apache.nutch.util.LogFormatter to not
>> insert itself as the default handler for the system.
>>
>>    I have been using Nutch for a year and have been waiting for a
>> version that I can embed into OpenEdit. The problem has been that
>> Nutch inserts itself as the formatter for the Java log system and that
>> interferes with OpenEdit logging.
>>
>>
>> ------------------------------------------------------------------------
>>
>> diff -Naur ../java/org/apache/nutch/util/LogFormatter.java
>> java/org/apache/nutch/util/LogFormatter.java
>> --- ../java/org/apache/nutch/util/LogFormatter.java    2006-03-31
>> 13:40:50.000000000 -0500
>> +++ java/org/apache/nutch/util/LogFormatter.java    2006-04-05
>> 16:27:59.000000000 -0400
>> @@ -16,13 +16,23 @@
>>  
>>  package org.apache.nutch.util;
>>  
>> -import java.util.logging.*;
>> -import java.io.*;
>> -import java.text.*;
>> +import java.io.ByteArrayOutputStream;
>> +import java.io.IOException;
>> +import java.io.PrintStream;
>> +import java.io.PrintWriter;
>> +import java.io.StringWriter;
>> +import java.text.FieldPosition;
>> +import java.text.SimpleDateFormat;
>>  import java.util.Date;
>> -
>> -/** Prints just the date and the log message. */
>> -
>> +import java.util.logging.Formatter;
>> +import java.util.logging.Level;
>> +import java.util.logging.LogRecord;
>> +import java.util.logging.Logger;
>> +
>> +/** Prints just the date and the log message. + *  This was also used
>> to stop processing as nutch crawls a web site
>> + *  [hidden email] changed this code to use a LogWrapper class
>> to catch severe errors
>> + * */
>>  public class LogFormatter extends Formatter {
>>    private static final String FORMAT = "yyMMdd HHmmss";
>>    private static final String NEWLINE =
>> System.getProperty("line.separator");
>> @@ -35,20 +45,27 @@
>>    private static boolean showTime = true;
>>    private static boolean showThreadIDs = false;
>>  
>> +  protected static LogFormatter sharedformatter =  new LogFormatter();
>> +  protected static SevereLogHandler sharedhandler =  new
>> SevereLogHandler(sharedformatter);
>> +
>> +  /*
>>    // install when this class is loaded
>>    static {
>>      Handler[] handlers = LogFormatter.getLogger("").getHandlers();
>>      for (int i = 0; i < handlers.length; i++) {
>> -      handlers[i].setFormatter(new LogFormatter());
>> +      handlers[i].setFormatter(sharedformatter);
>>        handlers[i].setLevel(Level.FINEST);
>>      }
>>    }
>> -
>> +  */
>>    /** Gets a logger and, as a side effect, installs this as the default
>>     * formatter. */
>>    public static Logger getLogger(String name) {
>>      // just referencing this class installs it
>> -    return Logger.getLogger(name);
>> +    Logger logr = Logger.getLogger(name);
>> +    logr.addHandler(sharedhandler);
>> +  
>> +    return logr;
>>    }
>>       /** When true, time is logged with each entry. */
>> @@ -60,7 +77,10 @@
>>    public static void setShowThreadIDs(boolean showThreadIDs) {
>>      LogFormatter.showThreadIDs = showThreadIDs;
>>    }
>> -
>> +  public void setLoggedSevere( boolean inSevere )
>> +  {
>> +      loggedSevere = inSevere;
>> +  }
>>    /**
>>     * Format the given LogRecord.
>>     * @param record the log record to be formatted.
>> diff -Naur ../java/org/apache/nutch/util/SevereLogHandler.java
>> java/org/apache/nutch/util/SevereLogHandler.java
>> --- ../java/org/apache/nutch/util/SevereLogHandler.java    1969-12-31
>> 19:00:00.000000000 -0500
>> +++ java/org/apache/nutch/util/SevereLogHandler.java    2006-04-05
>> 16:29:20.000000000 -0400
>> @@ -0,0 +1,46 @@
>> +/*
>> + * Created on Apr 5, 2006
>> + */
>> +package org.apache.nutch.util;
>> +
>> +import java.util.logging.Handler;
>> +import java.util.logging.Level;
>> +import java.util.logging.LogRecord;
>> +
>> +public class SevereLogHandler extends Handler
>> +{
>> +    protected LogFormatter fieldNutchFormatter;
>> +  
>> +    public SevereLogHandler(LogFormatter inFormatter)
>> +    {
>> +        setNutchFormatter(inFormatter);
>> +    }
>> +  
>> +    protected LogFormatter getNutchFormatter()
>> +    {
>> +        return fieldNutchFormatter;
>> +    }
>> +
>> +    protected void setNutchFormatter(LogFormatter inNutchFormatter)
>> +    {
>> +        fieldNutchFormatter = inNutchFormatter;
>> +    }
>> +
>> +    public void publish(LogRecord inRecord)
>> +    {
>> +        if ( inRecord.getLevel().intValue() == Level.SEVERE.intValue())
>> +        {
>> +            getNutchFormatter().setLoggedSevere(true);
>> +        }
>> +    }
>> +
>> +    public void flush()
>> +    {
>> +    }
>> +
>> +    public void close() throws SecurityException
>> +    {
>> +    }
>> +  
>> +
>> +}
>>  
>
>