Parse log file while with between lines relation - file

Question

I have long log file that contents looks like
2015-06-13 20:58:32,278 60157353 [Thread-1] DEBUG ccc - start PROC, will wait 30
2015-06-13 20:58:32,302 60157377 [Thread-1] DEBUG ccc - stoping PROC 0
2015-06-13 20:58:42,339 60167414 [Thread-1] DEBUG ccc - start PROC, will wait 30
2015-06-13 20:58:42,363 60167438 [Thread-1] DEBUG ccc - stoping PROC 0
2015-06-13 20:58:52,378 60177453 [Thread-1] DEBUG ccc - start PROC, will wait 30
2015-06-13 20:58:52,404 60177479 [Thread-1] DEBUG ccc - stoping PROC 0
2015-06-13 20:58:52,430 60177506 [Thread-1] DEBUG ccc - start PROC, will wait 30
I need to check time between start PROC and stoping PROC is not longer than 30 seconds.
Is it somehow possible do this with any log parser software?

Answer 1

Using a LogMX Parser, you can mark each start/stop couple as "Too long" (if there is more than 30s between start PROC and stoping PROC).
In the following Parser example, when the elapsed time is greater than 30s:
The user-defined log entry field named "TooLong" is set to "x" (else, it is empty) => can easily filter/sort/search using this field
The stoping PROC entry is marked as ERROR to appear in red => can quickly see it
Of course, you can adjust this code according to your needs.
To use this parser:
Copy the following code in a new file <LogMX_dir>/parsers/src/sample/parser/VicoParser.java
Compile it using Eclipse, IntelliJ IDEA, Maven, Gradle, or Ant using files in <LogMX_dir>/parsers (see LogMX documentation)
Add this Parser in LogMX using menu "Tools" > "Options" > "Parsers" > green "+" button > "Java class Parser" tab > choose <LogMX_dir>/parsers/classes/sample.parser/VicoParser
VicoParser.java:
package sample.parser;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.lightysoft.logmx.business.ParsedEntry;
import com.lightysoft.logmx.mgr.LogFileParser;
/**
* Sample LogMX Parser able to parse a log file with multi-line support, Absolute/Relative Date support,
* and detection of too-long elapsed time between too specific entries.<BR/>
*
* Log4j Pattern for this log format is:
* %d %-4r [%t] %-5p %c %x - %m%n
*
* Here is an example of log file suitable for this parser:<BR/>
* 2015-06-13 20:58:32,278 60157353 [Thread-1] DEBUG ccc - start PROC, will wait 30
* 2015-06-13 20:58:32,302 60157377 [Thread-1] DEBUG ccc - stoping PROC 0
* 2015-06-13 20:58:42,339 60167414 [Thread-1] DEBUG ccc - start PROC, will wait 30
* 2015-06-13 20:58:42,363 60167438 [Thread-1] DEBUG ccc - stoping PROC 0
* 2015-06-13 20:58:52,378 60177453 [Thread-1] DEBUG ccc - start PROC, will wait 30
* 2015-06-13 20:58:52,404 60177479 [Thread-1] DEBUG ccc - stoping PROC 0
* 2015-06-13 20:58:52,430 60177506 [Thread-1] DEBUG ccc - start PROC, will wait 30
*/
public class VicoParser extends LogFileParser {
/** Current parsed log entry */
private ParsedEntry entry = null;
/** Entry date format (this is Log4j default ISO-8601) */
private static SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss,SSS");
/** Mutex to avoid that multiple threads use the same Date formatter at the same time */
private final Object DATE_FORMATTER_MUTEX = new Object();
/** Pattern for entry begin */
private final static Pattern ENTRY_BEGIN_PATTERN = Pattern.compile(
// %d
"^(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d+?)\\s+?"
// %-4r [%t] %-5p
+ "(\\d+?)\\s+?\\[(.*?)\\]\\s+?(.*?)\\s+?"
// %c %x - %m
+ "(.*?) (.*?) - (.*)$");
/** Buffer for Entry message (improves performance for multi-lines entries) */
private StringBuilder entryMsgBuffer = null;
///////////// Elapsed-Time computation ////////////
/** Log entry message used for T0 (elapsed time calculation) */
private static final String LOG_MESSAGE_T0 = "start PROC";
/** Log entry message used for T1 (elapsed time calculation) */
private static final String LOG_MESSAGE_T1 = "stoping PROC";
/** Last encountered T0 entry */
private ParsedEntry prevT0Entry = null;
/** Max allowed time between entries, before raising "TooLong" flag */
private static final long MAXIMUM_DELTA_T = 30000L; // 30s (30,000 ms)
/////////////////////////////////////////////////////
/** Key of user-defined field "Timestamp" (internal, not displayed) */
private static final String EXTRA_FIELD_KEY__TIMESTAMP = "Timestamp";
/** Key of user-defined field "NDC" */
private static final String EXTRA_FIELD_KEY__NDC = "NDC";
/** Key of user-defined field "TooLong" */
private static final String EXTRA_FIELD_KEY__TOOLONG = "TooLong";
/** User-defined fields names */
private static final List<String> EXTRA_FIELDS_KEYS = Arrays.asList(EXTRA_FIELD_KEY__NDC,
EXTRA_FIELD_KEY__TOOLONG);
/**
* Returns the name of this parser
* #see com.lightysoft.logmx.mgr.LogFileParser#getParserName()
*/
#Override
public String getParserName() {
return "Vico Parser";
}
/**
* Returns the supported file type for this parser
* #see com.lightysoft.logmx.mgr.LogFileParser#getSupportedFileType()
*/
#Override
public String getSupportedFileType() {
return "Vico log files";
}
/**
* Process the new line of text read from file
* #see com.lightysoft.logmx.mgr.LogFileParser#parseLine(java.lang.String)
*/
#Override
protected void parseLine(String line) throws Exception {
// If end of file, records last entry if necessary, and exits
if (line == null) {
recordPreviousEntryIfExists();
return;
}
Matcher matcher = ENTRY_BEGIN_PATTERN.matcher(line);
if (matcher.matches()) {
// Record previous found entry if exists, then create a new one
prepareNewEntry();
entry.setDate(matcher.group(1));
entry.setThread(matcher.group(3));
entry.setLevel(matcher.group(4));
entry.setEmitter(matcher.group(5));
String logMsg = matcher.group(7);
// Save relative timestamp (in ms), for "getRelativeEntryDate()", but also to compute elapsed
// time between two specific log entries (faster than parsing complete absolute date)
long timestamp = Integer.parseInt(matcher.group(2), 10);
entryMsgBuffer.append(logMsg);
entry.getUserDefinedFields().put(EXTRA_FIELD_KEY__NDC, matcher.group(6)); // save NDC
entry.getUserDefinedFields().put(EXTRA_FIELD_KEY__TIMESTAMP, timestamp); // save Timestamp
if (logMsg.startsWith(LOG_MESSAGE_T0)) {
if (prevT0Entry != null) {
System.err.println("Warning: found [" + LOG_MESSAGE_T0 + "] not followed by ["
+ LOG_MESSAGE_T1 + "]");
}
prevT0Entry = entry;
} else if (logMsg.startsWith(LOG_MESSAGE_T1)) {
if (prevT0Entry == null) {
System.err.println("Warning: found [" + LOG_MESSAGE_T1 + "] not preceded by ["
+ LOG_MESSAGE_T0 + "]");
} else {
long prevT0 = (Long) prevT0Entry.getUserDefinedFields().get(
EXTRA_FIELD_KEY__TIMESTAMP);
if (timestamp - prevT0 > MAXIMUM_DELTA_T) {
entry.getUserDefinedFields().put(EXTRA_FIELD_KEY__TOOLONG, "x"); // Flag this entry as "TooLong"
prevT0Entry.getUserDefinedFields().put(EXTRA_FIELD_KEY__TOOLONG, "x"); // Flag this entry as "TooLong"
// Change log entry Level (note: cannot change Level of T0 entry because it has been already processed by LogMX)
entry.setLevel("ERROR");
}
prevT0Entry = null;
}
}
} else if (entry != null) {
entryMsgBuffer.append('\n').append(line); // appends this line to previous entry's text
}
}
/**
* Returns the ordered list of user-defined fields to display (given by their key), for each entry.
* #see com.lightysoft.logmx.mgr.LogFileParser#getUserDefinedFields()
*/
#Override
public List<String> getUserDefinedFields() {
return EXTRA_FIELDS_KEYS;
}
/**
* Returns a relative Date for the given entry
* #see com.lightysoft.logmx.mgr.LogFileParser#getRelativeEntryDate(com.lightysoft.logmx.business.ParsedEntry)
*/
#Override
public Date getRelativeEntryDate(ParsedEntry pEntry) throws Exception {
Long timestamp = (Long) pEntry.getUserDefinedFields().get(EXTRA_FIELD_KEY__TIMESTAMP);
return new Date(timestamp);
}
/**
* Returns the absolute Date for the given entry
* #see com.lightysoft.logmx.mgr.LogFileParser#getAbsoluteEntryDate(com.lightysoft.logmx.business.ParsedEntry)
*/
#Override
public Date getAbsoluteEntryDate(ParsedEntry pEntry) throws Exception {
synchronized (DATE_FORMATTER_MUTEX) { // Java date formatter is not thread-safe
return dateFormat.parse(pEntry.getDate());
}
}
/**
* Send to LogMX the current parsed log entry
* #throws Exception
*/
private void recordPreviousEntryIfExists() throws Exception {
if (entry != null) {
entry.setMessage(entryMsgBuffer.toString());
addEntry(entry);
}
}
/**
* Send to LogMX the current parsed log entry, then create a new one
* #throws Exception
*/
private void prepareNewEntry() throws Exception {
recordPreviousEntryIfExists();
entry = createNewEntry();
entryMsgBuffer = new StringBuilder(80);
entry.setUserDefinedFields(new HashMap<String, Object>(4));
}
}
And here is what I get:
Note: you can sort/filter log entries using the field named "TooLong" by clicking on its column (mouse left/middle button, or menu "Filter" > "Show filtering bar")