/*
 * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
 * agreements. See the NOTICE file distributed with this work for additional information regarding
 * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance with the License. You may obtain a
 * copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */
package org.apache.geode.internal.cache.tier.sockets;

import java.net.InetAddress;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicIntegerArray;

import org.apache.commons.lang.StringUtils;
import org.apache.logging.log4j.Logger;

import org.apache.geode.CancelException;
import org.apache.geode.SystemFailure;
import org.apache.geode.distributed.internal.DistributionConfig;
import org.apache.geode.distributed.internal.membership.InternalDistributedMember;
import org.apache.geode.internal.SystemTimer.SystemTimerTask;
import org.apache.geode.internal.Version;
import org.apache.geode.internal.cache.CacheClientStatus;
import org.apache.geode.internal.cache.IncomingGatewayStatus;
import org.apache.geode.internal.cache.InternalCache;
import org.apache.geode.internal.cache.TXId;
import org.apache.geode.internal.cache.TXManagerImpl;
import org.apache.geode.internal.concurrent.ConcurrentHashSet;
import org.apache.geode.internal.i18n.LocalizedStrings;
import org.apache.geode.internal.logging.LogService;
import org.apache.geode.internal.logging.LoggingThreadGroup;
import org.apache.geode.internal.logging.log4j.LocalizedMessage;

/**
 * Class <code>ClientHealthMonitor</code> is a server-side singleton that monitors the health of
 * clients by looking at their heartbeats. If too much time elapses between heartbeats, the monitor
 * determines that the client is dead and interrupts its threads.
 *
 * @since GemFire 4.2.3
 */
public class ClientHealthMonitor {
  private static final Logger logger = LogService.getLogger();
  public static final String CLIENT_HEALTH_MONITOR_INTERVAL_PROPERTY =
      "geode.client-health-monitor-interval";

  /**
   * The map of known clients
   */
  protected volatile Map _clientHeartbeats = Collections.EMPTY_MAP;

  /**
   * An object used to lock the map of known clients
   */
  final protected Object _clientHeartbeatsLock = new Object();

  /**
   * The map of known client threads
   */
  final protected Map _clientThreads;

  /**
   * An object used to lock the map of client threads
   */
  final private Object _clientThreadsLock = new Object();

  /**
   * THe GemFire <code>Cache</code>
   */
  private final InternalCache _cache;

  public int getMaximumTimeBetweenPings() {
    return maximumTimeBetweenPings;
  }

  private final int maximumTimeBetweenPings;

  /**
   * A thread that validates client connections
   */
  final private ClientHealthMonitorThread _clientMonitor;

  /**
   * The singleton <code>CacheClientNotifier</code> instance
   */
  static ClientHealthMonitor _instance;

  /**
   * Reference count in the event that multiple bridge servers are using the health monitor
   */

  private static int refCount = 0;

  /**
   * The interval between client monitor iterations
   */
  private final static long DEFAULT_CLIENT_MONITOR_INTERVAL_IN_MILLIS = 1000;

  private final CacheClientNotifierStats stats;

  /**
   * Used to track the number of handshakes in a VM primary use, license enforcement.
   *
   * note, these were moved from static fields in ServerConnection so that they will be cleaned up
   * when the client health monitor is shutdown.
   */
  private final HashMap cleanupTable = new HashMap();

  private final HashMap cleanupProxyIdTable = new HashMap();

  /**
   * Gives, version-wise, the number of clients connected to the cache servers in this cache, which
   * are capable of processing recieved deltas.
   * 
   * NOTE: It does not necessarily give the actual number of clients per version connected to the
   * cache servers in this cache.
   * 
   * @see CacheClientNotifier#addClientProxy(CacheClientProxy)
   */
  AtomicIntegerArray numOfClientsPerVersion = new AtomicIntegerArray(Version.HIGHEST_VERSION + 1);

  public long getMonitorInterval() {
    return monitorInterval;
  }

  private long monitorInterval;

  /**
   * Factory method to construct or return the singleton <code>ClientHealthMonitor</code> instance.
   * 
   * @param cache The GemFire <code>Cache</code>
   * @param maximumTimeBetweenPings The maximum time allowed between pings before determining the
   *        client has died and interrupting its sockets.
   * @return The singleton <code>ClientHealthMonitor</code> instance
   */
  public static ClientHealthMonitor getInstance(InternalCache cache, int maximumTimeBetweenPings,
      CacheClientNotifierStats stats) {
    createInstance(cache, maximumTimeBetweenPings, stats);
    return _instance;
  }

  /**
   * Factory method to return the singleton <code>ClientHealthMonitor</code> instance.
   * 
   * @return the singleton <code>ClientHealthMonitor</code> instance
   */
  public static ClientHealthMonitor getInstance() {
    return _instance;
  }

  /**
   * Shuts down the singleton <code>ClientHealthMonitor</code> instance.
   */
  public synchronized static void shutdownInstance() {
    refCount--;
    if (_instance == null)
      return;
    if (refCount > 0)
      return;
    _instance.shutdown();

    boolean interrupted = false; // Don't clear, let join fail if already interrupted
    try {
      if (_instance._clientMonitor != null) {
        _instance._clientMonitor.join();
      }
    } catch (InterruptedException e) {
      interrupted = true;
      if (logger.isDebugEnabled()) {
        logger.debug(":Interrupted joining with the ClientHealthMonitor Thread", e);
      }
    } finally {
      if (interrupted) {
        Thread.currentThread().interrupt();
      }
    }
    _instance = null;
    refCount = 0;
  }

  /**
   * Registers a new client to be monitored.
   * 
   * @param proxyID The id of the client to be registered
   */
  public void registerClient(ClientProxyMembershipID proxyID) {
    boolean registerClient = false;
    synchronized (_clientHeartbeatsLock) {
      Map oldClientHeartbeats = this._clientHeartbeats;
      if (!oldClientHeartbeats.containsKey(proxyID)) {
        Map newClientHeartbeats = new HashMap(oldClientHeartbeats);
        newClientHeartbeats.put(proxyID, Long.valueOf(System.currentTimeMillis()));
        this._clientHeartbeats = newClientHeartbeats;
        registerClient = true;
      }
    }

    if (registerClient) {
      if (this.stats != null) {
        this.stats.incClientRegisterRequests();
      }
      if (logger.isDebugEnabled()) {
        logger.debug(LocalizedMessage.create(
            LocalizedStrings.ClientHealthMonitor_CLIENTHEALTHMONITOR_REGISTERING_CLIENT_WITH_MEMBER_ID_0,
            new Object[] {proxyID}));
      }
    }
  }

  /**
   * Takes care of unregistering from the _clientHeatBeats map.
   * 
   * @param proxyID The id of the client to be unregistered
   */
  private void unregisterClient(ClientProxyMembershipID proxyID, boolean clientDisconnectedCleanly,
      Throwable clientDisconnectException) {
    boolean unregisterClient = false;
    synchronized (_clientHeartbeatsLock) {
      Map oldClientHeartbeats = this._clientHeartbeats;
      if (oldClientHeartbeats.containsKey(proxyID)) {
        unregisterClient = true;
        Map newClientHeartbeats = new HashMap(oldClientHeartbeats);
        newClientHeartbeats.remove(proxyID);
        this._clientHeartbeats = newClientHeartbeats;
      }
    }

    if (unregisterClient) {
      if (clientDisconnectedCleanly) {
        if (logger.isDebugEnabled()) {
          logger.debug(LocalizedMessage.create(
              LocalizedStrings.ClientHealthMonitor_CLIENTHEALTHMONITOR_UNREGISTERING_CLIENT_WITH_MEMBER_ID_0,
              new Object[] {proxyID}));
        }
      } else {
        logger.warn(LocalizedMessage.create(
            LocalizedStrings.ClientHealthMonitor_CLIENTHEALTHMONITOR_UNREGISTERING_CLIENT_WITH_MEMBER_ID_0_DUE_TO_1,
            new Object[] {proxyID, clientDisconnectException == null ? "Unknown reason"
                : clientDisconnectException.getLocalizedMessage()}));
      }
      if (this.stats != null) {
        this.stats.incClientUnRegisterRequests();
      }
      expireTXStates(proxyID);
    }
  }

  /**
   * Unregisters a client to be monitored.
   * 
   * @param proxyID The id of the client to be unregistered
   * @param acceptor non-null if the call is from a <code>ServerConnection</code> (as opposed to a
   *        <code>CacheClientProxy</code>).
   * @param clientDisconnectedCleanly Whether the client disconnected cleanly or crashed
   */
  public void unregisterClient(ClientProxyMembershipID proxyID, AcceptorImpl acceptor,
      boolean clientDisconnectedCleanly, Throwable clientDisconnectException) {
    unregisterClient(proxyID, clientDisconnectedCleanly, clientDisconnectException);
    // Unregister any CacheClientProxy instances associated with this member id
    // if this method was invoked from a ServerConnection and the client did
    // not disconnect cleanly.
    if (acceptor != null) {
      CacheClientNotifier ccn = acceptor.getCacheClientNotifier();
      if (ccn != null) {
        try {
          ccn.unregisterClient(proxyID, clientDisconnectedCleanly);
        } catch (CancelException ignore) {
        }
      }
    }
  }

  private final Set<TXId> scheduledToBeRemovedTx =
      Boolean.getBoolean(DistributionConfig.GEMFIRE_PREFIX + "trackScheduledToBeRemovedTx")
          ? new ConcurrentHashSet<TXId>() : null;

  /**
   * provide a test hook to track client transactions to be removed
   */
  public Set<TXId> getScheduledToBeRemovedTx() {
    return scheduledToBeRemovedTx;
  }

  /**
   * expire the transaction states for the given client. This uses the transactionTimeToLive setting
   * that is inherited from the TXManagerImpl. If that setting is non-positive we expire the states
   * immediately
   * 
   * @param proxyID
   */
  private void expireTXStates(ClientProxyMembershipID proxyID) {
    final TXManagerImpl txMgr = (TXManagerImpl) this._cache.getCacheTransactionManager();
    final Set<TXId> txids =
        txMgr.getTransactionsForClient((InternalDistributedMember) proxyID.getDistributedMember());
    if (this._cache.isClosed()) {
      return;
    }
    long timeout = txMgr.getTransactionTimeToLive() * 1000;
    if (!txids.isEmpty()) {
      if (logger.isDebugEnabled()) {
        logger.debug("expiring {} transaction contexts for {} timeout={}", txids.size(), proxyID,
            timeout / 1000);
      }

      if (timeout <= 0) {
        txMgr.removeTransactions(txids, true);
      } else {
        if (scheduledToBeRemovedTx != null)
          scheduledToBeRemovedTx.addAll(txids);
        SystemTimerTask task = new SystemTimerTask() {
          @Override
          public void run2() {
            txMgr.removeTransactions(txids, true);
            if (scheduledToBeRemovedTx != null)
              scheduledToBeRemovedTx.removeAll(txids);
          }
        };
        this._cache.getCCPTimer().schedule(task, timeout);
      }
    }
  }

  public void removeAllConnectionsAndUnregisterClient(ClientProxyMembershipID proxyID,
      Throwable t) {
    // Remove all connections
    cleanupClientThreads(proxyID, false);

    unregisterClient(proxyID, false, t);
  }

  /**
   * Adds a <code>ServerConnection</code> to the client's processing threads
   * 
   * @param proxyID The membership id of the client to be updated
   * @param connection The thread processing client requests
   */
  public void addConnection(ClientProxyMembershipID proxyID, ServerConnection connection) {
    // logger.info("ClientHealthMonitor: Adding " + connection + " to
    // client with member id " + proxyID);
    synchronized (_clientThreadsLock) {
      Set serverConnections = (Set) this._clientThreads.get(proxyID);
      if (serverConnections == null) {
        serverConnections = new HashSet();
        this._clientThreads.put(proxyID, serverConnections);
      }
      serverConnections.add(connection);
      // logger.info("ClientHealthMonitor: The client with member id " +
      // proxyID + " contains " + serverConnections.size() + " threads");
    }
  }

  /**
   * Removes a <code>ServerConnection</code> from the client's processing threads
   * 
   * @param proxyID The id of the client to be updated
   * @param connection The thread processing client requests
   */
  public void removeConnection(ClientProxyMembershipID proxyID, ServerConnection connection) {
    // logger.info("ClientHealthMonitor: Removing " + connection + " from
    // client with member id " + proxyID);
    synchronized (_clientThreadsLock) {
      Set serverConnections = (Set) this._clientThreads.get(proxyID);
      if (serverConnections != null) { // fix for bug 35343
        serverConnections.remove(connection);
        // logger.info("ClientHealthMonitor: The client with member id " +
        // proxyID + " contains " + serverConnections.size() + " threads");
        if (serverConnections.isEmpty()) {
          // logger.info("ClientHealthMonitor: The client with member id "
          // + proxyID + " is being removed since it contains 0 threads");
          this._clientThreads.remove(proxyID);
        }
      }
    }
  }

  /**
   * Processes a received ping for a client.
   * 
   * @param proxyID The id of the client from which the ping was received
   */
  public void receivedPing(ClientProxyMembershipID proxyID) {
    if (this._clientMonitor == null) {
      return;
    }
    if (logger.isTraceEnabled()) {
      logger.trace("ClientHealthMonitor: Received ping from client with member id {}", proxyID);
    }
    synchronized (_clientHeartbeatsLock) {
      if (!this._clientHeartbeats.containsKey(proxyID)) {
        registerClient(proxyID);
      } else {
        this._clientHeartbeats.put(proxyID, Long.valueOf(System.currentTimeMillis()));
      }
    }
  }

  /**
   * Returns modifiable map (changes do not effect this class) of client membershipID to connection
   * count. This is different from the map contained in this class as here the key is client
   * membershipID & not the the proxyID. It is to be noted that a given client can have multiple
   * proxies.
   * 
   * @param filterProxies Set identifying the Connection proxies which should be fetched. These
   *        ConnectionProxies may be from same client member or different. If it is null this would
   *        mean to fetch the Connections of all the ConnectionProxy objects.
   */
  public Map getConnectedClients(Set filterProxies) {
    Map map = new HashMap(); // KEY=proxyID, VALUE=connectionCount (Integer)
    synchronized (_clientThreadsLock) {
      Iterator connectedClients = this._clientThreads.entrySet().iterator();
      while (connectedClients.hasNext()) {
        Map.Entry entry = (Map.Entry) connectedClients.next();
        ClientProxyMembershipID proxyID = (ClientProxyMembershipID) entry.getKey();// proxyID
                                                                                   // includes FQDN
        if (filterProxies == null || filterProxies.contains(proxyID)) {
          String membershipID = null;
          Set connections = (Set) entry.getValue();
          int socketPort = 0;
          InetAddress socketAddress = null;
          /// *
          Iterator serverConnections = connections.iterator();
          // Get data from one.
          while (serverConnections.hasNext()) {
            ServerConnection sc = (ServerConnection) serverConnections.next();
            socketPort = sc.getSocketPort();
            socketAddress = sc.getSocketAddress();
            membershipID = sc.getMembershipID();
            break;
          }
          // */
          int connectionCount = connections.size();
          String clientString = null;
          if (socketAddress == null) {
            clientString = "client member id=" + membershipID;
          } else {
            clientString = "host name=" + socketAddress.toString() + " host ip="
                + socketAddress.getHostAddress() + " client port=" + socketPort
                + " client member id=" + membershipID;
          }
          Object[] data = null;
          data = (Object[]) map.get(membershipID);
          if (data == null) {
            map.put(membershipID, new Object[] {clientString, Integer.valueOf(connectionCount)});
          } else {
            data[1] = Integer.valueOf(((Integer) data[1]).intValue() + connectionCount);
          }
          /*
           * Note: all client addresses are same... Iterator serverThreads = ((Set)
           * entry.getValue()).iterator(); while (serverThreads.hasNext()) { ServerConnection
           * connection = (ServerConnection) serverThreads.next(); InetAddress clientAddress =
           * connection.getClientAddress(); logger.severe("getConnectedClients: proxyID=" + proxyID
           * + " clientAddress=" + clientAddress + " FQDN=" + clientAddress.getCanonicalHostName());
           * }
           */
        }
      }

    }
    return map;
  }

  /**
   * This method returns the CacheClientStatus for all the clients that are connected to this
   * server. This method returns all clients irrespective of whether subscription is enabled or not.
   * 
   * @return Map of ClientProxyMembershipID against CacheClientStatus objects.
   */
  public Map getStatusForAllClients() {
    Map result = new HashMap();
    synchronized (_clientThreadsLock) {
      Iterator connectedClients = this._clientThreads.entrySet().iterator();
      while (connectedClients.hasNext()) {
        Map.Entry entry = (Map.Entry) connectedClients.next();
        ClientProxyMembershipID proxyID = (ClientProxyMembershipID) entry.getKey();
        CacheClientStatus cci = new CacheClientStatus(proxyID);
        Set connections = (Set) this._clientThreads.get(proxyID);
        if (connections != null) {
          String memberId = null;
          Iterator connectionsIterator = connections.iterator();
          while (connectionsIterator.hasNext()) {
            ServerConnection sc = (ServerConnection) connectionsIterator.next();
            if (sc.isClientServerConnection()) {
              memberId = sc.getMembershipID(); // each ServerConnection has the same member id
              cci.setMemberId(memberId);
              cci.setNumberOfConnections(connections.size());
              result.put(proxyID, cci);
              break;
            }
          }
        }
      }
    }
    return result;
  }

  public void fillInClientInfo(Map allClients) {
    // The allClients parameter includes only actual clients (not remote
    // gateways). This monitor will include remote gateway connections,
    // so weed those out.
    synchronized (_clientThreadsLock) {
      Iterator allClientsIterator = allClients.entrySet().iterator();
      while (allClientsIterator.hasNext()) {
        Map.Entry entry = (Map.Entry) allClientsIterator.next();
        ClientProxyMembershipID proxyID = (ClientProxyMembershipID) entry.getKey();// proxyID
                                                                                   // includes FQDN
        CacheClientStatus cci = (CacheClientStatus) entry.getValue();
        Set connections = (Set) this._clientThreads.get(proxyID);
        if (connections != null) {
          String memberId = null;
          cci.setNumberOfConnections(connections.size());
          List socketPorts = new ArrayList();
          List socketAddresses = new ArrayList();
          Iterator connectionsIterator = connections.iterator();
          while (connectionsIterator.hasNext()) {
            ServerConnection sc = (ServerConnection) connectionsIterator.next();
            socketPorts.add(Integer.valueOf(sc.getSocketPort()));
            socketAddresses.add(sc.getSocketAddress());
            memberId = sc.getMembershipID(); // each ServerConnection has the
                                             // same member id
          }
          cci.setMemberId(memberId);
          cci.setSocketPorts(socketPorts);
          cci.setSocketAddresses(socketAddresses);
        }
      }
    }
  }

  public Map getConnectedIncomingGateways() {
    Map connectedIncomingGateways = new HashMap();
    synchronized (_clientThreadsLock) {
      Iterator connectedClients = this._clientThreads.entrySet().iterator();
      while (connectedClients.hasNext()) {
        Map.Entry entry = (Map.Entry) connectedClients.next();
        ClientProxyMembershipID proxyID = (ClientProxyMembershipID) entry.getKey();
        Set connections = (Set) entry.getValue();
        Iterator connectionsIterator = connections.iterator();
        while (connectionsIterator.hasNext()) {
          ServerConnection sc = (ServerConnection) connectionsIterator.next();
          if (sc.getCommunicationMode().isWAN()) {
            IncomingGatewayStatus status = new IncomingGatewayStatus(proxyID.getDSMembership(),
                sc.getSocketAddress(), sc.getSocketPort());
            connectedIncomingGateways.put(proxyID.getDSMembership(), status);
          }
        }
      }
    }
    return connectedIncomingGateways;
  }

  protected boolean cleanupClientThreads(ClientProxyMembershipID proxyID, boolean timedOut) {
    boolean result = false;
    Set serverConnections = null;
    synchronized (this._clientThreadsLock) {
      serverConnections = (Set) this._clientThreads.remove(proxyID);
      // It is ok to modify the set after releasing the sync
      // because it has been removed from the map while holding
      // the sync.
    } // end sync here to fix bug 37576 and 36740
    {
      if (serverConnections != null) { // fix for bug 35343
        result = true;
        // logger.warn("Terminating " + serverConnections.size() + " connections");
        for (Iterator it = serverConnections.iterator(); it.hasNext();) {
          ServerConnection serverConnection = (ServerConnection) it.next();
          // logger.warn("Terminating " + serverConnection);
          serverConnection.handleTermination(timedOut);
        }
      }
    }
    return result;
  }

  protected boolean isAnyThreadProcessingMessage(ClientProxyMembershipID proxyID) {
    boolean processingMessage = false;
    synchronized (this._clientThreadsLock) {
      Set serverConnections = (Set) this._clientThreads.get(proxyID);
      if (serverConnections != null) {
        for (Iterator it = serverConnections.iterator(); it.hasNext();) {
          ServerConnection serverConnection = (ServerConnection) it.next();
          if (serverConnection.isProcessingMessage()) {
            processingMessage = true;
            break;
          }
        }
      }
    }
    return processingMessage;
  }

  protected void validateThreads(ClientProxyMembershipID proxyID) {
    Set serverConnections = null;
    synchronized (this._clientThreadsLock) {
      serverConnections = (Set) this._clientThreads.get(proxyID);
      if (serverConnections != null) {
        serverConnections = new HashSet(serverConnections);
      }
    }
    // release sync and operation on copy to fix bug 37675
    if (serverConnections != null) {
      for (Iterator it = serverConnections.iterator(); it.hasNext();) {
        ServerConnection serverConnection = (ServerConnection) it.next();
        if (serverConnection.hasBeenTimedOutOnClient()) {
          logger.warn(LocalizedMessage.create(
              LocalizedStrings.ClientHealtMonitor_0_IS_BEING_TERMINATED_BECAUSE_ITS_CLIENT_TIMEOUT_OF_1_HAS_EXPIRED,
              new Object[] {serverConnection,
                  Integer.valueOf(serverConnection.getClientReadTimeout())}));
          try {
            serverConnection.handleTermination(true);
            // Not all the code in a ServerConnection correctly
            // handles interrupt. In particular it is possible to be doing
            // p2p distribution and to have sent a message to one peer but
            // to never send it to another due to interrupt.
            // serverConnection.interruptOwner();
          } finally {
            // Just to be sure we clean it up.
            // This call probably isn't needed.
            removeConnection(proxyID, serverConnection);
          }
        }
      }
    }
  }

  /**
   * Returns the map of known clients.
   * 
   * @return the map of known clients
   */
  public Map getClientHeartbeats() {
    return this._clientHeartbeats;
  }

  /**
   * Shuts down the singleton <code>CacheClientNotifier</code> instance.
   */
  protected synchronized void shutdown() {
    // Stop the client monitor
    if (this._clientMonitor != null) {
      this._clientMonitor.stopMonitoring();
    }
  }

  /**
   * Creates the singleton <code>CacheClientNotifier</code> instance.
   * 
   * @param cache The GemFire <code>Cache</code>
   * @param maximumTimeBetweenPings The maximum time allowed between pings before determining the
   */
  protected static synchronized void createInstance(InternalCache cache,
      int maximumTimeBetweenPings, CacheClientNotifierStats stats) {
    refCount++;
    if (_instance != null) {
      return;
    }
    _instance = new ClientHealthMonitor(cache, maximumTimeBetweenPings, stats);
  }

  /**
   *
   * Constructor.
   * 
   * @param cache The GemFire <code>Cache</code>
   * @param maximumTimeBetweenPings The maximum time allowed between pings before determining the
   */
  private ClientHealthMonitor(InternalCache cache, int maximumTimeBetweenPings,
      CacheClientNotifierStats stats) {
    // Set the Cache
    this._cache = cache;
    this.maximumTimeBetweenPings = maximumTimeBetweenPings;

    // Initialize the client threads map
    this._clientThreads = new HashMap();

    this.monitorInterval = Long.getLong(CLIENT_HEALTH_MONITOR_INTERVAL_PROPERTY,
        DEFAULT_CLIENT_MONITOR_INTERVAL_IN_MILLIS);
    logger.debug("Setting monitorInterval to {}", this.monitorInterval);

    if (maximumTimeBetweenPings > 0) {
      if (logger.isDebugEnabled()) {
        logger.debug("{}: Initializing client health monitor thread", this);
      }
      this._clientMonitor = new ClientHealthMonitorThread(maximumTimeBetweenPings);
      this._clientMonitor.start();
    } else {
      // LOG:CONFIG: changed from config to info
      logger.info(LocalizedMessage.create(
          LocalizedStrings.ClientHealthMonitor_CLIENT_HEALTH_MONITOR_THREAD_DISABLED_DUE_TO_MAXIMUMTIMEBETWEENPINGS_SETTING__0,
          maximumTimeBetweenPings));
      this._clientMonitor = null;
    }

    this.stats = stats;
  }

  /**
   * Returns a brief description of this <code>ClientHealthMonitor</code>
   *
   * @since GemFire 5.1
   */
  @Override
  public String toString() {
    return "ClientHealthMonitor@" + Integer.toHexString(System.identityHashCode(this));
  }

  public Map getCleanupProxyIdTable() {
    return cleanupProxyIdTable;
  }

  public Map getCleanupTable() {
    return cleanupTable;
  }

  public int getNumberOfClientsAtVersion(Version version) {
    return numOfClientsPerVersion.get(version.ordinal());
  }

  public int getNumberOfClientsAtOrAboveVersion(Version version) {
    int number = 0;
    for (int i = version.ordinal(); i < numOfClientsPerVersion.length(); i++) {
      number += numOfClientsPerVersion.get(i);
    }
    return number;
  }

  public boolean hasDeltaClients() {
    return getNumberOfClientsAtOrAboveVersion(Version.GFE_61) > 0;
  }

  /**
   * Class <code>ClientHealthMonitorThread</code> is a <code>Thread</code> that verifies all clients
   * are still alive.
   */
  class ClientHealthMonitorThread extends Thread {

    /**
     * The maximum time allowed between pings before determining the client has died and
     * interrupting its sockets.
     */
    final protected int _maximumTimeBetweenPings;

    /**
     * Whether the monitor is stopped
     */
    protected volatile boolean _isStopped = false;

    /**
     * Constructor.
     * 
     * @param maximumTimeBetweenPings The maximum time allowed between pings before determining the
     *        client has died and interrupting its sockets
     */
    protected ClientHealthMonitorThread(int maximumTimeBetweenPings) {
      super(LoggingThreadGroup.createThreadGroup("ClientHealthMonitor Thread Group", logger),
          "ClientHealthMonitor Thread");
      setDaemon(true);

      // Set the client connection timeout
      this._maximumTimeBetweenPings = maximumTimeBetweenPings;
      // LOG:CONFIG: changed from config to info
      logger.info(LocalizedMessage.create(
          LocalizedStrings.ClientHealthMonitor_CLIENTHEALTHMONITORTHREAD_MAXIMUM_ALLOWED_TIME_BETWEEN_PINGS_0,
          this._maximumTimeBetweenPings));
      if (maximumTimeBetweenPings == 0) {
        if (logger.isDebugEnabled()) {
          logger.debug("zero ping interval detected", new Exception(
              LocalizedStrings.ClientHealthMonitor_STACK_TRACE_0.toLocalizedString()));
        }
      }
    }

    /**
     * Notifies the monitor to stop monitoring.
     */
    protected synchronized void stopMonitoring() {
      if (logger.isDebugEnabled()) {
        logger.debug("{}: Stopping monitoring", ClientHealthMonitor.this);
      }
      this._isStopped = true;
      this.interrupt();
      if (logger.isDebugEnabled()) {
        logger.debug("{}: Stopped dispatching", ClientHealthMonitor.this);
      }
    }

    /**
     * Returns whether the dispatcher is stopped
     * 
     * @return whether the dispatcher is stopped
     */
    protected boolean isStopped() {
      return this._isStopped;
    }

    /**
     * Runs the monitor by iterating the map of clients and testing the latest ping time received
     * against the current time.
     */
    @Override
    public void run() {
      if (logger.isDebugEnabled()) {
        logger.debug("{}: Beginning to monitor clients", ClientHealthMonitor.this);
      }

      while (!this._isStopped) {
        SystemFailure.checkFailure();
        try {
          Thread.sleep(monitorInterval);
          if (logger.isTraceEnabled()) {
            logger.trace("Monitoring {} client(s)", getClientHeartbeats().size());
          }
          // logger.warning("Monitoring " + getClientHeartbeats().size() +
          // " client(s).");

          // Get the current time
          long currentTime = System.currentTimeMillis();
          if (logger.isTraceEnabled()) {
            logger.trace("{} starting sweep at {}", ClientHealthMonitor.this, currentTime);
          }

          // Iterate through the clients and verify that they are all still
          // alive
          for (Iterator i = getClientHeartbeats().entrySet().iterator(); i.hasNext();) {
            Map.Entry entry = (Map.Entry) i.next();
            ClientProxyMembershipID proxyID = (ClientProxyMembershipID) entry.getKey();
            // Validate all ServerConnection threads. If a thread has been
            // processing a message for more than the socket timeout time,
            // close it it since the client will have timed out and resent.
            validateThreads(proxyID);

            Long latestHeartbeatValue = (Long) entry.getValue();
            // Compare the current value with the current time if it is not null
            // If it is null, that means that the client was just registered
            // and has not done a heartbeat yet.
            if (latestHeartbeatValue != null) {
              long latestHeartbeat = latestHeartbeatValue.longValue();
              if (logger.isTraceEnabled()) {
                logger.trace(
                    "{} ms have elapsed since the latest heartbeat for client with member id {}",
                    (currentTime - latestHeartbeat), proxyID);
              }

              if ((currentTime - latestHeartbeat) > this._maximumTimeBetweenPings) {
                // This client has been idle for too long. Determine whether
                // any of its ServerConnection threads are currently processing
                // a message. If so, let it go. If not, disconnect it.
                if (isAnyThreadProcessingMessage(proxyID)) {
                  if (logger.isDebugEnabled()) {
                    logger.debug(
                        "Monitoring client with member id {}. It has been {} ms since the latest heartbeat. This client would have been terminated but at least one of its threads is processing a message.",
                        entry.getKey(), (currentTime - latestHeartbeat));
                  }
                } else {
                  if (cleanupClientThreads(proxyID, true)) {
                    logger.warn(LocalizedMessage.create(
                        LocalizedStrings.ClientHealthMonitor_MONITORING_CLIENT_WITH_MEMBER_ID_0_IT_HAD_BEEN_1_MS_SINCE_THE_LATEST_HEARTBEAT_MAX_INTERVAL_IS_2_TERMINATED_CLIENT,
                        new Object[] {entry.getKey(), currentTime - latestHeartbeat,
                            this._maximumTimeBetweenPings}));
                  }
                }
              } else {
                if (logger.isTraceEnabled()) {
                  logger.trace(
                      "Monitoring client with member id {}. It has been {} ms since the latest heartbeat. This client is healthy.",
                      entry.getKey(), (currentTime - latestHeartbeat));
                }
                // logger.warning("Monitoring client with member id " +
                // entry.getKey() + ". It has been " + (currentTime -
                // latestHeartbeat) + " ms since the latest heartbeat. This
                // client is healthy.");
              }
            }
          }
        } catch (InterruptedException e) {
          // no need to reset the bit; we're exiting
          if (this._isStopped) {
            break;
          }
          logger.warn(LocalizedMessage
              .create(LocalizedStrings.ClientHealthMonitor_UNEXPECTED_INTERRUPT_EXITING), e);
          break;
        } catch (Exception e) {
          // An exception occurred while monitoring the clients. If the monitor
          // is not stopped, log it and continue processing.
          if (!this._isStopped) {
            logger.fatal(LocalizedMessage.create(
                LocalizedStrings.ClientHealthMonitor_0_AN_UNEXPECTED_EXCEPTION_OCCURRED,
                ClientHealthMonitor.this), e);
          }
        }
      } // while
    }
  } // ClientHealthMonitorThread
}
