/*
 * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
 * agreements. See the NOTICE file distributed with this work for additional information regarding
 * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance with the License. You may obtain a
 * copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */
package org.apache.geode.internal.logging;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;

import org.apache.geode.SystemFailure;
import org.apache.geode.internal.Assert;
import org.apache.geode.internal.ExitCode;
import org.apache.geode.internal.i18n.LocalizedStrings;

/**
 * This program merges entries from multiple GemFire log files (those written using a
 * {@link org.apache.geode.i18n.LogWriterI18n} together, sorting them by their timestamp. Note that
 * this program assumes that the entries in the individual log files are themselves sorted by
 * timestamp.
 * <p>
 *
 * MergeLogFiles has several command line options:<br>
 * <br>
 * <b>-pids</b> tells the program to look for hydra-style naming and pick out process identifiers,
 * then use these to distinguish log entries instead of full log file names.<br>
 * <br>
 * <b>-dirCount x</b> when <i>-pids</i> is not being used, or it is being used and a process ID
 * can't be found, the <i>-dirCount</i> option instructs the program as to how many parent directory
 * names should be included in the name of each log file.<br>
 * <br>
 * <b>-noblanks</b> tells the program to suppress blank lines found in log files<br>
 * <br>
 * <b>-align</b> tells the program to align log content not having timestamps with log content that
 * does have timestamps.<br>
 * <br>
 * <b>-threads</b> tells the program to attempt to make use of multiple CPUs
 * <p>
 *
 * The <i>-pids</i> option will cause a file nickname table to be built and emitted at the beginning
 * of the merged log files. Nicknames are of the form pid-x, where <i>pid</i> is the process ID that
 * emitted the log entry, and <i>x</i> is the position of this log file in the ordered set of log
 * files created by that process.
 * <p>
 * 
 * Normally, one log file reader is created per log file, and they are iterated over repeatedly to
 * find and write the next earliest timestamped line to the output stream. Text without a timestamp
 * is associated with the prior timestamped line in the file.
 * <p>
 * 
 * The <i>-threads</i> option will cause the program to also create threads for each reader that are
 * backed by bounded {@link BlockingQueue queues}, as outlined in the diagram below. This can
 * consume more memory, so it is wise to increase the Xmx of the java virtual machine if you are
 * going to use this option.
 * <p>
 *
 * <CENTER> <IMG SRC="{@docRoot}/javadoc-images/merge-log-files.gif" WIDTH="353" HEIGHT="246"/>
 * </CENTER>
 *
 * @see SortLogFile
 * @see LogFileParser
 *
 *
 *
 * @since GemFire 2.0 (-pids, -threads, -align, and -noblanks added in 5.1)
 */
public class MergeLogFiles {
  private static PrintStream out = System.out;
  private static PrintStream err = System.err;

  /**
   * Merges the log files from a given set of <code>InputStream</code>s into a
   * <code>PrinWriter</code>.
   *
   * @param logFiles The log files to be merged
   * @param logFileNames The names of the log files to be printed in the merged log
   * @param mergedFile Where the merged logs are printed to
   * 
   * @return Whether or not problems occurred while merging the log files.
   *
   * @throws IllegalArgumentException If the length of <code>logFiles</code> is not the same as the
   *         length of <code>logFileNames</code>
   */
  public static boolean mergeLogFiles(InputStream[] logFiles, String[] logFileNames,
      PrintWriter mergedFile) {
    return mergeLogFiles(logFiles, logFileNames, mergedFile, false, false, false, new LinkedList());
  }

  /**
   * Merges the log files from a given set of <code>InputStream</code>s into a
   * <code>PrinWriter</code>.
   *
   * @param logFiles The log files to be merged
   * @param logFileNames The names of the log files to be printed in the merged log
   * @param mergedFile Where the merged logs are printed to
   * @param tabOut Whether to align non-timestamped lines with timestamped lines
   * @param suppressBlanks Whether to omit blank lines
   * @param patterns Regular expression patterns that lines must match to be included
   * @return Whether or not problems occurred while merging the log files.
   *
   * @throws IllegalArgumentException If the length of <code>logFiles</code> is not the same as the
   *         length of <code>logFileNames</code>
   */
  public static boolean mergeLogFiles(InputStream[] logFiles, String[] logFileNames,
      PrintWriter mergedFile, boolean tabOut, boolean suppressBlanks, boolean multithreaded,
      List<String> patterns) {
    return Sorter.mergeLogFiles(logFiles, logFileNames, mergedFile, tabOut, suppressBlanks,
        multithreaded, patterns);
  }


  // ////////////////// Main Program ////////////////////

  /**
   * Prints usage information about this program
   */
  private static void usage(String s) {
    // note that we don't document the -pids switch because it is tailored
    // to how hydra works and would not be useful for customers
    err.println("\n** " + s + "\n");
    err.println(LocalizedStrings.MergeLogFiles_USAGE.toLocalizedString()
        + ": java MergeLogFiles [(directory | logFile)]+");
    err.println("-dirCount n      "
        + LocalizedStrings.MergeLogFiles_NUMBER_OF_PARENT_DIRS_TO_PRINT.toLocalizedString());
    err.println("-mergeFile file  "
        + LocalizedStrings.MergeLogFiles_FILE_IN_WHICH_TO_PUT_MERGED_LOGS.toLocalizedString());
    err.println("-pids            "
        + LocalizedStrings.MergeLogFiles_SEARCH_FOR_PIDS_IN_FILE_NAMES_AND_USE_THEM_TO_IDENTIFY_FILES
            .toLocalizedString());
    err.println(
        "-align           " + LocalizedStrings.MergeLogFiles_ALIGN_NONTIMESTAMPED_LINES_WITH_OTHERS
            .toLocalizedString());
    err.println("-noblanks        "
        + LocalizedStrings.MergeLogFiles_SUPPRESS_OUTPUT_OF_BLANK_LINES.toLocalizedString());
    err.println("-threaded        "
        + LocalizedStrings.MergeLogFiles_USE_MULTITHREADING_TO_TAKE_ADVANTAGE_OF_MULTIPLE_CPUS
            .toLocalizedString());
    // err.println("-regex pattern Case-insensitive search for a regular expression.");
    // err.println(" May be used multiple times. Use Java regular ");
    // err.println(" expression syntax (see java.util.regex.Pattern).");
    err.println("");
    err.println(
        LocalizedStrings.MergeLogFiles_MERGES_MULTIPLE_GEMFIRE_LOG_FILES_AND_SORTS_THEM_BY_TIMESTAMP
            .toLocalizedString());
    err.println(
        LocalizedStrings.MergeLogFiles_THE_MERGED_LOG_FILE_IS_WRITTEN_TO_SYSTEM_OUT_OR_A_FILE
            .toLocalizedString());
    err.println("");
    err.println(
        LocalizedStrings.MergeLogFiles_IF_A_DIRECTORY_IS_SPECIFIED_ALL_LOG_FILES_IN_THAT_DIRECTORY_ARE_MERGED
            .toLocalizedString());
    err.println("");
    ExitCode.FATAL.doSystemExit();
  }

  /**
   * Find all of the .log files in the given directory
   * 
   * @param dirName directory to search
   * @return all of the .log files found (Files)
   */
  static ArrayList<File> getLogFiles(String dirName) {
    ArrayList<File> result = new ArrayList<>();

    File dir = new File(dirName);
    File names[] = dir.listFiles();
    if (names != null) {
      for (final File name : names) {
        String n = name.getAbsolutePath();
        if (n.endsWith(".log") || n.endsWith(".log.gz")) {
          result.add(name);
        }
      } // for
    }
    return result;
  }

  public static void main(String[] args) throws IOException {
    File mergeFile = null;
    ArrayList files = new ArrayList();
    List nickNames = null;
    int dirCount = 0;
    boolean findPIDs = false;
    boolean tabOut = false;
    boolean suppressBlanks = false;
    boolean multithreaded = false;
    List<String> patterns = new LinkedList();

    // Parse command line
    for (int i = 0; i < args.length; i++) {
      if (args[i].equals("-align")) {
        tabOut = true;
      } else if (args[i].equals("-noblanks")) {
        suppressBlanks = true;
      } else if (args[i].equals("-pids")) {
        findPIDs = true;
      } else if (args[i].equals("-threaded")) {
        multithreaded = true;
      } else if (args[i].equals("-regex")) {
        if (i + 1 >= args.length) {
          usage("missing pattern for -regex option");
        }
        patterns.add(args[i + 1]);
        i++;
      } else if (args[i].equals("-dirCount")) {
        if (++i >= args.length) {
          usage(LocalizedStrings.MergeLogFiles_MISSING_NUMBER_OF_PARENT_DIRECTORIES
              .toLocalizedString());
        }

        try {
          dirCount = Integer.parseInt(args[i]);

        } catch (NumberFormatException ex) {
          usage(LocalizedStrings.MergeLogFiles_NOT_A_NUMBER_0.toLocalizedString(args[i]));
        }

      } else if (args[i].equals("-mergeFile")) {
        if (++i >= args.length) {
          usage(LocalizedStrings.MergeLogFiles_MISSING_MERGE_FILE_NAME.toLocalizedString());
        }

        mergeFile = new File(args[i]);

      } else {
        File file = new File(args[i]);
        if (!file.exists()) {
          usage(LocalizedStrings.MergeLogFiles_FILE_0_DOES_NOT_EXIST.toLocalizedString(file));
        }

        files.add(file.getAbsoluteFile());
      }
    } // for
    if (files.isEmpty()) {
      usage(LocalizedStrings.MergeLogFiles_MISSING_FILENAME.toLocalizedString());
    }

    // Expand directory names found in list
    ArrayList expandedFiles = new ArrayList();
    for (int i = 0; i < files.size(); i++) {
      File f = (File) files.get(i);
      String n = f.getAbsolutePath();
      if (!f.exists()) {
        usage(LocalizedStrings.MergeLogFiles_FILE_0_DOES_NOT_EXIST.toLocalizedString(n));
      }
      if (f.isFile()) {
        expandedFiles.add(f);
        continue;
      }
      if (f.isDirectory()) {
        ArrayList moreFiles = getLogFiles(n);
        expandedFiles.addAll(moreFiles);
        continue;
      }
      usage(LocalizedStrings.MergeLogFiles_FILE_0_IS_NEITHER_A_FILE_NOR_A_DIRECTORY
          .toLocalizedString(n));
    }
    Collections.sort(expandedFiles);
    files = expandedFiles;

    // Create output stream
    PrintStream ps;
    if (mergeFile != null) {
      ps = new PrintStream(new FileOutputStream(mergeFile), true);

    } else {
      ps = out;
    }
    PrintWriter mergedFile = new PrintWriter(ps, true);

    ps.println("Merged files (count = " + expandedFiles.size() + ") input list:");
    for (int i = 0; i < expandedFiles.size(); i++) {
      ps.println("  " + expandedFiles.get(i));
    }
    ps.println("");

    if (findPIDs) {
      nickNames = findPIDs(files, mergedFile);
    }

    InputStream[] logFiles = new InputStream[files.size()];
    String[] logFileNames = new String[files.size()];
    for (int i = 0; i < files.size(); i++) {
      File file = (File) files.get(i);
      logFiles[i] = new FileInputStream(file);

      if (findPIDs && (nickNames.get(i) != null)) {
        if (file.getCanonicalPath().toLowerCase().endsWith("gz")) {
          logFileNames[i] = (String) nickNames.get(i) + ".gz";
        } else {
          logFileNames[i] = (String) nickNames.get(i);
        }
      } else {
        StringBuffer sb = new StringBuffer();
        File parent = file.getParentFile();
        for (int j = 0; j < dirCount && parent != null; j++) {
          String parentName = parent.getName() + "/";
          // don't add dot-slash
          if (parentName.equals("./")) {
            parent = null;
          } else {
            sb.insert(0, parentName);
            parent = parent.getParentFile();
          }
        }
        sb.append(file.getName());

        logFileNames[i] = sb.toString();
      }
    }

    mergeLogFiles(logFiles, logFileNames, mergedFile, tabOut, suppressBlanks, multithreaded,
        patterns);

    ExitCode.NORMAL.doSystemExit();
  }

  ////////////////////// Inner Classes //////////////////////

  /**
   * hydra log files usually have the process's PID in their path name. This method extracts the PID
   * number and assigns the corresponding File a nickname using the PID and the position of the File
   * in the list of those also having this PID.
   * <p>
   * e.g., bgexec32414_1043.log --> 1043-1<br>
   * gemfire_1043/system.log --> 1043-2<br>
   * gemfire_1043/system_01_00.log --> 1043-3<br>
   */
  private static ArrayList findPIDs(ArrayList files, PrintWriter output) {
    int pidTable[] = new int[files.size()];
    int pidTableCounter[] = new int[pidTable.length];
    ArrayList nickNames = new ArrayList();
    char sep = File.separatorChar;

    // System.out.println("findPids() invoked");

    for (Iterator it = files.iterator(); it.hasNext();) {
      File f = (File) it.next();
      String name = f.getPath();
      // System.out.println("considering " + name);

      String slashdotslash = "" + sep + "." + sep;
      int startIdx = name.lastIndexOf(slashdotslash);

      // get rid of the parent directories and any /./ in the path
      if (startIdx > 0) {
        name = name.substring(startIdx + slashdotslash.length());
      }

      startIdx = name.lastIndexOf(sep);

      // first see if there's a number at the end of the file's directory name
      if (startIdx > 0) {
        startIdx--;
        char c = name.charAt(startIdx);
        if (!('0' <= c && c <= '9')) {
          // System.out.println("no number found in directory name");
          startIdx = 0;
        } else {
          // see if this is a hydra-generated test directory name, like
          // parReg-0504-161349
          int testIdx = startIdx - 1;
          while (testIdx > 0 && '0' <= name.charAt(testIdx) && name.charAt(testIdx) <= '9') {
            testIdx--;
          }
          if (testIdx < 1 || name.charAt(testIdx) == '-') {
            startIdx = 0;
          }
          // System.out.println("using directory name: '" + name.substring(0, startIdx+1) + "'");
        }
      }

      // if there's no number in the directory name, use the file name
      if (startIdx <= 0) {
        startIdx = name.length() - 1;
        if (startIdx > 6 && name.charAt(startIdx) == 'z' && name.charAt(startIdx - 1) == 'g'
            && name.charAt(startIdx - 2) == '.' && name.charAt(startIdx - 3) == 'g'
            && name.charAt(startIdx - 4) == 'o' && name.charAt(startIdx - 5) == 'l'
            && name.charAt(startIdx - 6) == '.') {
          startIdx -= 7;
        } else if (startIdx > 3 && name.charAt(startIdx) == 'g' && name.charAt(startIdx - 1) == 'o'
            && name.charAt(startIdx - 2) == 'l' && name.charAt(startIdx - 3) == '.') {
          startIdx -= 4;
          // System.out.println("using file name: '" + name.substring(0,startIdx+1) + "'");
        }
        // else {
        // System.out.println("could not find a PID");
        // }
      }

      // find the string of numbers at the end of the test area and use it
      // as a PID
      String PID = null;
      for (int i = startIdx; i >= 0; i--) {
        char c = name.charAt(i);
        // System.out.println("charAt("+i+")="+c);
        if (!('0' <= c && c <= '9')) {
          if (i < (name.length() - 1)) { // have a number
            // there's a number - assume it's a PID if it's not zero
            PID = name.substring(i + 1, startIdx + 1);
            // System.out.println("parsing '" + PID + "'");
            try {
              int iPID = Integer.valueOf(PID).intValue();
              if (iPID > 0) {
                // System.out.println("Found PID " + iPID);
                int p = 0;
                // find the PID in the table of those seen so far, or assign it
                // a new slot. increment the number of files for this PID and
                // assign a nickname for the file
                for (; p < pidTable.length; p++) {
                  if (pidTable[p] == 0) {
                    pidTable[p] = iPID;
                    pidTableCounter[p] = 1;
                    break;
                  } else if (pidTable[p] == iPID) {
                    pidTableCounter[p]++;
                    break;
                  }
                }
                Assert.assertTrue(p < pidTableCounter.length);
                nickNames.add("" + iPID + "-" + pidTableCounter[p]);
                output.println("nickname " + iPID + "-" + pidTableCounter[p] + ": " + name);
              } else {
                nickNames.add(null);
              }
            } catch (NumberFormatException nfe) {
              nickNames.add(null);
            }
          } // have a number
          else {
            nickNames.add(null);
          }
          break;
        } // not a digit
      } // for(i)
    }
    return nickNames;
  }

  /** interface for threaded and non-threaded reader classes */
  static interface Reader {
    public LogFileParser.LogEntry peek();

    public LogFileParser.LogEntry poll();

    public String getFileName();

    public void setUniqueId(int id);

    public int getUniqueId();
  }


  /**
   * Thread that reads an entry from a GemFire log file and adds it a bounded queue. The entries are
   * consumed by a {@link MergeLogFiles.Sorter}.
   */
  static class NonThreadedReader implements Reader {

    /** The maximum size of the entry queue */
    // private static int QUEUE_CAPACITY = 1000;

    //////////////////// Instance Methods ////////////////////

    /** The log file */
    private BufferedReader logFile;

    /** The name of the log file */
    private String logFileName;

    /** whether to suppress blank lines */
    // private boolean suppressBlanks;

    /** whether to align non-timestamped lines with timestamped lines */
    // private boolean tabOut;

    private LogFileParser parser;

    private LogFileParser.LogEntry nextEntry;

    private List<Pattern> patterns;

    //////////////////// Constructors ////////////////////

    /**
     * Creates a new <code>Reader</code> that reads from the given log file with the given name.
     * Invoking this constructor will start this reader thread.
     * 
     * @param patterns java regular expressions that an entry must match one or more of
     */
    public NonThreadedReader(InputStream logFile, String logFileName, ThreadGroup group,
        boolean tabOut, boolean suppressBlanks, List<Pattern> patterns) {
      // super(group, "Reader for " + ((logFileName != null) ? logFileName : logFile.toString()));
      if (logFileName.endsWith(".gz")) {
        try {
          this.logFile = new BufferedReader(new InputStreamReader(new GZIPInputStream(logFile)));
        } catch (IOException e) {
          System.err.println(logFileName + " does not appear to be in gzip format");
          this.logFile = new BufferedReader(new InputStreamReader(logFile));
        }
      } else {
        this.logFile = new BufferedReader(new InputStreamReader(logFile));
      }
      this.logFileName = logFileName;
      this.patterns = patterns;
      // this.suppressBlanks = suppressBlanks;
      // this.tabOut = tabOut;
      this.parser = new LogFileParser(this.logFileName, this.logFile, tabOut, suppressBlanks);
    }

    /** returns the file name being read */
    public String getFileName() {
      return this.logFileName;
    }

    /** unique identifier, used for sorting instead of file name */
    private int uniqueId;

    /** set the unique identifier for this reader */
    public void setUniqueId(int id) {
      uniqueId = id;
    }

    /** retrieve the unique identifier for this reader */
    public int getUniqueId() {
      return uniqueId;
    }

    /**
     * Peeks at the oldest log entry read from the log file, waits for a log entry to become
     * available.
     *
     * @return <code>null</code> if interrupted while waiting
     */
    public synchronized LogFileParser.LogEntry peek() {
      while (this.nextEntry == null) {
        try {
          nextEntry = parser.getNextEntry();
          if (nextEntry == null) {
            return null;
          }
          if (!nextEntry.isLast() && !patternMatch(nextEntry)) {
            continue;
          }
        } catch (IOException ioe) {
          ioe.printStackTrace(System.err);
        }
      }
      return nextEntry;
    }


    /** return true if the entry matches one or more regex patterns */
    private boolean patternMatch(LogFileParser.LogEntry entry) {
      if (this.patterns == null || this.patterns.isEmpty()) {
        return true;
      }
      for (Pattern p : patterns) {
        if (p.matcher(entry.getContents()).matches()) {
          return true;
        }
      }
      return false;
    }

    /**
     * Removes the old log entry read from the log file
     */
    public LogFileParser.LogEntry poll() {
      LogFileParser.LogEntry returnValue = null;
      if (this.nextEntry != null) {
        returnValue = this.nextEntry;
        nextEntry = null;
      } else {
        while (returnValue == null) {
          try {
            returnValue = parser.getNextEntry();
            if (!returnValue.isLast() && !patternMatch(returnValue)) {
              returnValue = null;
              continue;
            }
          } catch (IOException ioe) {
            ioe.printStackTrace(System.err);
            break;
          }
        }
      }
      return returnValue;
    }
  }

  /**
   * Thread that reads an entry from a GemFire log file and adds it a bounded queue. The entries are
   * consumed by a {@link MergeLogFiles.Sorter}.
   */
  static class ThreadedReader extends Thread implements Reader {

    /** The maximum size of the entry queue */
    private static int QUEUE_CAPACITY = 1000;

    //////////////////// Instance Methods ////////////////////

    /** The log file */
    private BufferedReader logFile;

    /** The name of the log file */
    private String logFileName;

    /** The queue containing log entries */
    private BlockingQueue queue;

    /** whether to suppress blank lines */
    private boolean suppressBlanks;

    /** whether to align non-timestamped lines with timestamped lines */
    private boolean tabOut;

    private List<Pattern> patterns;

    //////////////////// Constructors ////////////////////

    /**
     * Creates a new <code>Reader</code> that reads from the given log file with the given name.
     * Invoking this constructor will start this reader thread.
     * 
     * @param patterns TODO
     *
     * @see #run
     */
    public ThreadedReader(InputStream logFile, String logFileName, ThreadGroup group,
        boolean tabOut, boolean suppressBlanks, List<Pattern> patterns) {
      // super(group, "Reader for " + ((logFileName != null) ? logFileName : logFile.toString()));
      super(group, LocalizedStrings.MergeLogFiles_LOG_FILE_READER.toLocalizedString());
      if (logFileName.endsWith(".gz")) {
        try {
          this.logFile = new BufferedReader(new InputStreamReader(new GZIPInputStream(logFile)));
        } catch (IOException e) {
          System.err.println(logFileName + " does not appear to be in gzip format");
          this.logFile = new BufferedReader(new InputStreamReader(logFile));
        }
      } else {
        this.logFile = new BufferedReader(new InputStreamReader(logFile));
      }
      this.logFileName = logFileName;
      this.queue = new LinkedBlockingQueue(QUEUE_CAPACITY);
      // new UnsharedMessageQueue(QUEUE_CAPACITY,
      // (75 * QUEUE_CAPACITY) / 100);
      this.suppressBlanks = suppressBlanks;
      this.tabOut = tabOut;
      this.patterns = patterns;
      this.start();
    }

    /** returns the file name being read */
    public String getFileName() {
      return this.logFileName;
    }

    /** unique identifier, used for sorting instead of file name */
    private int uniqueId;

    /** set the unique identifier for this reader */
    public void setUniqueId(int id) {
      uniqueId = id;
    }

    /** retrieve the unique identifier for this reader */
    public int getUniqueId() {
      return uniqueId;
    }

    /**
     * Reads the log file and places {@link LogFileParser.LogEntry} objects into the queue. When it
     * is finished, it places a <code>LogEntry</code> that whose
     * {@link LogFileParser.LogEntry#isLast isLast} method will return <code>true</code>.
     */
    @Override
    public void run() {
      LogFileParser parser =
          new LogFileParser(this.logFileName, this.logFile, tabOut, suppressBlanks);

      try {
        while (true) {
          SystemFailure.checkFailure();
          LogFileParser.LogEntry entry = parser.getNextEntry();
          if (entry.isLast() || patternMatch(entry)) {
            this.queue.put(entry);

            synchronized (this) {
              this.notify();
            }
          }
          if (entry.isLast()) {
            break;
          }
        }

      } catch (IOException ex) {
        ex.printStackTrace(System.err);

      } catch (InterruptedException ex) {
        Thread.currentThread().interrupt();
      }
    }

    /** return true if the entry matches one or more regex patterns */
    private boolean patternMatch(LogFileParser.LogEntry entry) {
      if (this.patterns == null || this.patterns.isEmpty()) {
        return true;
      }
      for (Pattern p : patterns) {
        if (p.matcher(entry.getContents()).matches()) {
          return true;
        }
      }
      return false;
    }

    /**
     * Peeks at the oldest log entry read from the log file, waits for a log entry to become
     * available.
     *
     * @return <code>null</code> if interrupted while waiting
     */
    public LogFileParser.LogEntry peek() {
      // out.println(this.getName() + " size " + this.queue.size());
      LogFileParser.LogEntry entry = (LogFileParser.LogEntry) this.queue.peek();
      if (entry == null) {
        synchronized (this) {
          entry = (LogFileParser.LogEntry) this.queue.peek();
          while (entry == null) {
            boolean interrupted = Thread.interrupted();
            try {
              this.wait();
              entry = (LogFileParser.LogEntry) this.queue.peek();
            } catch (InterruptedException e) {
              interrupted = true;
            } finally {
              if (interrupted) {
                Thread.currentThread().interrupt();
              }
            }
          } // while
        } // synchronized
      }
      return entry;
    }

    /**
     * Removes the old log entry read from the log file
     */
    public LogFileParser.LogEntry poll() {
      return (LogFileParser.LogEntry) this.queue.poll();
    }
  }

  /**
   * A thread group that contains the reader threads and logs uncaught exceptions to standard error.
   */
  static class ReaderGroup extends ThreadGroup {
    /** Did an uncaught exception occur? */
    private boolean exceptionOccurred;

    ReaderGroup(String groupName) {
      super(groupName);
      this.exceptionOccurred = false;
    }

    @Override
    public void uncaughtException(Thread t, Throwable e) {
      if (e instanceof VirtualMachineError) {
        SystemFailure.setFailure((VirtualMachineError) e); // don't throw
      }
      this.exceptionOccurred = true;
      System.err.println(LocalizedStrings.MergeLogFiles_EXCEPTION_IN_0.toLocalizedString(t));
      e.printStackTrace(System.err);
    }

    /**
     * Returns whether or not an uncaught exception occurred in one of the threads in this group.
     */
    public boolean exceptionOccurred() {
      return this.exceptionOccurred;
    }
  }

  /**
   * Examines the {@link MergeLogFiles.Reader#peek oldest entry} in each log file and writes it to a
   * merged log file.
   */
  static class Sorter {

    /**
     * Merges the log files from a given set of <code>InputStream</code>s into a
     * <code>PrintWriter</code>.
     *
     * @param logFiles The log files to be merged
     * @param logFileNames The names of the log files to be printed in the merged log
     * @param mergedFile Where the merged logs are printed to
     * @param tabOut Whether to align non-timestamped lines with others
     * @param suppressBlanks Whether to suppress output of blank lines
     * @param patterns TODO
     * @return Whether or not problems occurred while merging the log files.
     *
     * @throws IllegalArgumentException If the length of <code>logFiles</code> is not the same as
     *         the length of <code>logFileNames</code>
     */
    public static boolean mergeLogFiles(InputStream[] logFiles, String[] logFileNames,
        PrintWriter mergedFile, boolean tabOut, boolean suppressBlanks, boolean multithreaded,
        List<String> patterns) {
      if (logFiles.length != logFileNames.length) {
        throw new IllegalArgumentException(
            LocalizedStrings.MergeLogFiles_NUMBER_OF_LOG_FILES_0_IS_NOT_THE_SAME_AS_THE_NUMBER_OF_LOG_FILE_NAMES_1
                .toLocalizedString(new Object[] {Integer.valueOf(logFiles.length),
                    Integer.valueOf(logFileNames.length)}));
      }

      List<Pattern> compiledPatterns = new LinkedList<Pattern>();
      for (String pattern : patterns) {
        compiledPatterns.add(Pattern.compile(pattern, Pattern.CASE_INSENSITIVE));
      }

      // First start the Reader threads
      ReaderGroup group =
          new ReaderGroup(LocalizedStrings.MergeLogFiles_READER_THREADS.toLocalizedString());
      Collection readers = new ArrayList(logFiles.length);
      for (int i = 0; i < logFiles.length; i++) {
        if (multithreaded) {
          readers.add(new ThreadedReader(logFiles[i], logFileNames[i], group, tabOut,
              suppressBlanks, compiledPatterns));
        } else {
          readers.add(new NonThreadedReader(logFiles[i], logFileNames[i], group, tabOut,
              suppressBlanks, compiledPatterns));
        }
      }

      // Merge the log files together
      Reader lastOldest = null;

      // sort readers by their next time-stamp
      Set sorted = sortReaders(readers);

      while (!readers.isEmpty()) {

        Reader oldest = null;
        Iterator sortedIt = sorted.iterator();
        if (!sortedIt.hasNext()) {
          break;
        }
        oldest = (Reader) sortedIt.next();
        sortedIt.remove();

        String nextReaderTimestamp = null;
        Reader nextInLine = null;
        if (sortedIt.hasNext()) {
          nextInLine = (Reader) sortedIt.next();
          nextReaderTimestamp = nextInLine.peek().getTimestamp();
        }


        // if we've switched to a different reader, emit a blank line
        // for readability
        if (oldest != lastOldest) {
          mergedFile.println();
          lastOldest = oldest;
        }

        LogFileParser.LogEntry entry = null;
        // write until we hit the next file's time-stamp
        do {
          entry = oldest.peek();
          String timestamp = entry.getTimestamp();

          if (nextReaderTimestamp != null) {
            if (nextReaderTimestamp.compareTo(timestamp) < 0) {
              sorted.add(oldest);
              entry = null;
              break;
            }
          }

          entry = oldest.poll();
          entry.writeTo(mergedFile);

        } while (!entry.isLast());

        if (entry != null && entry.isLast()) {
          readers.remove(oldest);
        }
      }

      return group.exceptionOccurred();
    }

    private static Set sortReaders(Collection readers) {
      Set sorted = new TreeSet(new ReaderComparator());
      int uniqueId = 1;
      for (Iterator iter = readers.iterator(); iter.hasNext();) {
        Reader reader = (Reader) iter.next();
        if (reader == null) {
          continue;
        }
        reader.setUniqueId(uniqueId++);
        sorted.add(reader);
      }
      return sorted;
    }

  }

  protected static class ReaderComparator implements Comparator {

    public int compare(Object o1, Object o2) {
      Reader reader1 = (Reader) o1;
      int id1 = reader1.getUniqueId();
      Reader reader2 = (Reader) o2;
      int id2 = reader2.getUniqueId();
      LogFileParser.LogEntry entry1 = reader1.peek();
      LogFileParser.LogEntry entry2 = reader2.peek();
      if (entry1 == null) {
        if (entry2 == null) {
          if (id1 < id2) {
            return -1;
          }
          // IDs are unique, so no need for == test
          return 1;
        }
        // sort readers with no entries before readers with entries so they'll
        // be removed quickly
        return -1;
      }
      if (entry2 == null) {
        return 1;
      }
      String timestamp1 = entry1.getTimestamp();
      String timestamp2 = entry2.getTimestamp();
      int compare = timestamp1.compareTo(timestamp2);
      if (compare == 0) {
        if (id1 < id2) {
          return -1;
        }
        return 1;
      }
      return compare;
    }
  }

}


