/*
 * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
 * agreements. See the NOTICE file distributed with this work for additional information regarding
 * copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance with the License. You may obtain a
 * copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software distributed under the License
 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
 * or implied. See the License for the specific language governing permissions and limitations under
 * the License.
 */
package org.apache.geode.internal.tcp;

import static org.apache.geode.distributed.ConfigurationProperties.SECURITY_PEER_AUTH_INIT;

import java.io.BufferedInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InterruptedIOException;
import java.io.OutputStream;
import java.net.ConnectException;
import java.net.Inet6Address;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.SocketException;
import java.net.SocketTimeoutException;
import java.nio.ByteBuffer;
import java.nio.channels.CancelledKeyException;
import java.nio.channels.ClosedChannelException;
import java.nio.channels.ClosedSelectorException;
import java.nio.channels.SocketChannel;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;

import org.apache.logging.log4j.Logger;

import org.apache.geode.CancelException;
import org.apache.geode.SystemFailure;
import org.apache.geode.cache.CacheClosedException;
import org.apache.geode.distributed.DistributedMember;
import org.apache.geode.distributed.DistributedSystemDisconnectedException;
import org.apache.geode.distributed.internal.ClusterDistributionManager;
import org.apache.geode.distributed.internal.ConflationKey;
import org.apache.geode.distributed.internal.DMStats;
import org.apache.geode.distributed.internal.DirectReplyProcessor;
import org.apache.geode.distributed.internal.DistributionConfig;
import org.apache.geode.distributed.internal.DistributionConfigImpl;
import org.apache.geode.distributed.internal.DistributionManager;
import org.apache.geode.distributed.internal.DistributionMessage;
import org.apache.geode.distributed.internal.DistributionStats;
import org.apache.geode.distributed.internal.ReplyException;
import org.apache.geode.distributed.internal.ReplyMessage;
import org.apache.geode.distributed.internal.ReplyProcessor21;
import org.apache.geode.distributed.internal.ReplySender;
import org.apache.geode.distributed.internal.direct.DirectChannel;
import org.apache.geode.distributed.internal.membership.InternalDistributedMember;
import org.apache.geode.distributed.internal.membership.MembershipManager;
import org.apache.geode.i18n.StringId;
import org.apache.geode.internal.Assert;
import org.apache.geode.internal.ByteArrayDataInput;
import org.apache.geode.internal.DSFIDFactory;
import org.apache.geode.internal.InternalDataSerializer;
import org.apache.geode.internal.SystemTimer;
import org.apache.geode.internal.SystemTimer.SystemTimerTask;
import org.apache.geode.internal.Version;
import org.apache.geode.internal.i18n.LocalizedStrings;
import org.apache.geode.internal.logging.LogService;
import org.apache.geode.internal.logging.LoggingThreadGroup;
import org.apache.geode.internal.logging.log4j.AlertAppender;
import org.apache.geode.internal.logging.log4j.LocalizedMessage;
import org.apache.geode.internal.net.SocketCreator;
import org.apache.geode.internal.tcp.MsgReader.Header;
import org.apache.geode.internal.util.concurrent.ReentrantSemaphore;

/**
 * Connection is a socket holder that sends and receives serialized message objects. A Connection
 * may be closed to preserve system resources and will automatically be reopened when it's needed.
 *
 * @since GemFire 2.0
 */
public class Connection implements Runnable {
  private static final Logger logger = LogService.getLogger();

  private static final int INITIAL_CAPACITY =
      Integer.getInteger("p2p.readerBufferSize", 32768).intValue();
  private static int P2P_CONNECT_TIMEOUT;
  private static boolean IS_P2P_CONNECT_TIMEOUT_INITIALIZED = false;

  public static final int NORMAL_MSG_TYPE = 0x4c;
  public static final int CHUNKED_MSG_TYPE = 0x4d; // a chunk of one logical msg
  public static final int END_CHUNKED_MSG_TYPE = 0x4e; // last in a series of chunks
  public static final int DIRECT_ACK_BIT = 0x20;

  public static final int MSG_HEADER_SIZE_OFFSET = 0;
  public static final int MSG_HEADER_TYPE_OFFSET = 4;
  public static final int MSG_HEADER_ID_OFFSET = 5;
  public static final int MSG_HEADER_BYTES = 7;

  /**
   * Small buffer used for send socket buffer on receiver connections and receive buffer on sender
   * connections.
   */
  public static final int SMALL_BUFFER_SIZE =
      Integer.getInteger(DistributionConfig.GEMFIRE_PREFIX + "SMALL_BUFFER_SIZE", 4096).intValue();

  /** counter to give connections a unique id */
  private static AtomicLong idCounter = new AtomicLong(1);

  /** string used as the reason for initiating suspect processing */
  public static final String INITIATING_SUSPECT_PROCESSING =
      "member unexpectedly shut down shared, unordered connection";

  /** the table holding this connection */
  private final ConnectionTable owner;

  private final TCPConduit conduit;

  /**
   * Set to false once run() is terminating. Using this instead of Thread.isAlive as the reader
   * thread may be a pooled thread.
   */
  private volatile boolean isRunning = false;

  /** true if connection is a shared resource that can be used by more than one thread */
  private boolean sharedResource;

  public boolean isSharedResource() {
    return this.sharedResource;
  }

  /** The idle timeout timer task for this connection */
  private SystemTimerTask idleTask;

  private static final ThreadLocal isReaderThread = new ThreadLocal();

  public static void makeReaderThread() {
    // mark this thread as a reader thread
    makeReaderThread(true);
  }

  private static void makeReaderThread(boolean v) {
    isReaderThread.set(v);
  }

  // return true if this thread is a reader thread
  public static boolean isReaderThread() {
    Object o = isReaderThread.get();
    if (o == null) {
      return false;
    } else {
      return ((Boolean) o).booleanValue();
    }
  }

  private int getP2PConnectTimeout() {
    if (IS_P2P_CONNECT_TIMEOUT_INITIALIZED)
      return P2P_CONNECT_TIMEOUT;
    String connectTimeoutStr = System.getProperty("p2p.connectTimeout");
    if (connectTimeoutStr != null) {
      P2P_CONNECT_TIMEOUT = Integer.parseInt(connectTimeoutStr);
    } else {
      P2P_CONNECT_TIMEOUT = 6 * this.conduit.getDM().getConfig().getMemberTimeout();
    }
    IS_P2P_CONNECT_TIMEOUT_INITIALIZED = true;
    return P2P_CONNECT_TIMEOUT;
  }

  /**
   * If true then readers for thread owned sockets will send all messages on thread owned senders.
   * Even normally unordered msgs get send on TO socks.
   */
  private static final boolean DOMINO_THREAD_OWNED_SOCKETS =
      Boolean.getBoolean("p2p.ENABLE_DOMINO_THREAD_OWNED_SOCKETS");

  private static final ThreadLocal isDominoThread = new ThreadLocal();

  // return true if this thread is a reader thread
  public static boolean tipDomino() {
    if (DOMINO_THREAD_OWNED_SOCKETS) {
      // mark this thread as one who wants to send ALL on TO sockets
      ConnectionTable.threadWantsOwnResources();
      isDominoThread.set(Boolean.TRUE);
      return true;
    } else {
      return false;
    }
  }

  public static boolean isDominoThread() {
    Object o = isDominoThread.get();
    if (o == null) {
      return false;
    } else {
      return ((Boolean) o).booleanValue();
    }
  }

  /** the socket entrusted to this connection */
  private final Socket socket;

  /** the non-NIO output stream */
  OutputStream output;

  /** output stream/channel lock */
  private final Object outLock = new Object();

  /** the ID string of the conduit (for logging) */
  String conduitIdStr;

  /** Identifies the java group member on the other side of the connection. */
  InternalDistributedMember remoteAddr;

  /**
   * Identifies the version of the member on the other side of the connection.
   */
  Version remoteVersion;

  /**
   * True if this connection was accepted by a listening socket. This makes it a receiver. False if
   * this connection was explicitly created by a connect call. This makes it a sender.
   */
  private final boolean isReceiver;

  /**
   * count of how many unshared p2p-readers removed from the original action this thread is. For
   * instance, server-connection -> owned p2p reader (count 0) -> owned p2p reader (count 1) ->
   * owned p2p reader (count 2). This shows up in thread names as "DOM #x" (domino #x)
   */
  private static ThreadLocal<Integer> dominoCount = new ThreadLocal<Integer>() {
    @Override
    protected Integer initialValue() {
      return 0;
    }
  };

  /**
   * How long to wait if receiver will not accept a message before we go into queue mode.
   *
   * @since GemFire 4.2.2
   */
  private int asyncDistributionTimeout = 0;

  /**
   * How long to wait, with the receiver not accepting any messages, before kicking the receiver out
   * of the distributed system. Ignored if asyncDistributionTimeout is zero.
   *
   * @since GemFire 4.2.2
   */
  private int asyncQueueTimeout = 0;

  /**
   * How much queued data we can have, with the receiver not accepting any messages, before kicking
   * the receiver out of the distributed system. Ignored if asyncDistributionTimeout is zero.
   * Canonicalized to bytes (property file has it as megabytes
   *
   * @since GemFire 4.2.2
   */
  private long asyncMaxQueueSize = 0;

  /**
   * True if an async queue is already being filled.
   */
  private volatile boolean asyncQueuingInProgress = false;

  /**
   * Maps ConflatedKey instances to ConflatedKey instance. Note that even though the key and value
   * for an entry is the map will always be "equal" they will not always be "==".
   */
  private final Map conflatedKeys = new HashMap();

  // NOTE: LinkedBlockingQueue has a bug in which removes from the queue
  // cause future offer to increase the size without adding anything to the queue.
  // So I've changed from this backport class to a java.util.LinkedList
  private final LinkedList outgoingQueue = new LinkedList();

  /**
   * Number of bytes in the outgoingQueue. Used to control capacity.
   */
  private long queuedBytes = 0;

  /** used for async writes */
  Thread pusherThread;

  /**
   * The maximum number of concurrent senders sending a message to a single recipient.
   */
  private static final int MAX_SENDERS = Integer
      .getInteger("p2p.maxConnectionSenders", DirectChannel.DEFAULT_CONCURRENCY_LEVEL).intValue();
  /**
   * This semaphore is used to throttle how many threads will try to do sends on this connection
   * concurrently. A thread must acquire this semaphore before it is allowed to start serializing
   * its message.
   */
  private final Semaphore senderSem = new ReentrantSemaphore(MAX_SENDERS);

  /** Set to true once the handshake has been read */
  volatile boolean handshakeRead = false;
  volatile boolean handshakeCancelled = false;

  private volatile int replyCode = 0;

  private static final byte REPLY_CODE_OK = (byte) 69;
  private static final byte REPLY_CODE_OK_WITH_ASYNC_INFO = (byte) 70;

  private final Object handshakeSync = new Object();

  /** message reader thread */
  private volatile Thread readerThread;

  /** whether the reader thread is, or should be, running */
  volatile boolean stopped = true;

  /** set to true once a close begins */
  private final AtomicBoolean closing = new AtomicBoolean(false);

  volatile boolean readerShuttingDown = false;

  /** whether the socket is connected */
  volatile boolean connected = false;

  /**
   * Set to true once a connection finishes its constructor
   */
  volatile boolean finishedConnecting = false;

  volatile boolean accessed = true;
  volatile boolean socketInUse = false;
  volatile boolean timedOut = false;

  /**
   * task for detecting ack timeouts and issuing alerts
   */
  private SystemTimer.SystemTimerTask ackTimeoutTask;

  /**
   * millisecond clock at the time message transmission started, if doing forced-disconnect
   * processing
   */
  long transmissionStartTime;

  /** ack wait timeout - if socketInUse, use this to trigger SUSPECT processing */
  private long ackWaitTimeout;

  /** ack severe alert timeout - if socketInUse, use this to send alert */
  private long ackSATimeout;

  /**
   * other connections participating in the current transmission. we notify them if ackSATimeout
   * expires to keep all members from generating alerts when only one is slow
   */
  List ackConnectionGroup;

  /** name of thread that we're currently performing an operation in (may be null) */
  String ackThreadName;

  /** the buffer used for NIO message receipt */
  ByteBuffer nioInputBuffer;

  /** the length of the next message to be dispatched */
  int nioMessageLength;

  /** the type of message being received */
  byte nioMessageType;

  /** used to lock access to destreamer data */
  private final Object destreamerLock = new Object();

  /** caches a msg destreamer that is currently not being used */
  MsgDestreamer idleMsgDestreamer;

  /**
   * used to map a msgId to a MsgDestreamer which are used for destreaming chunked messages using
   * nio
   */
  HashMap destreamerMap;

  boolean directAck;

  short nioMsgId;

  /** whether the length of the next message has been established */
  boolean nioLengthSet = false;

  /** is this connection used for serial message delivery? */
  boolean preserveOrder = false;

  /** number of messages sent on this connection */
  private long messagesSent;

  /** number of messages received on this connection */
  private long messagesReceived;

  /** unique ID of this connection (remote if isReceiver==true) */
  private volatile long uniqueId;

  private int sendBufferSize = -1;
  private int recvBufferSize = -1;

  private void setSendBufferSize(Socket sock) {
    setSendBufferSize(sock, this.owner.getConduit().tcpBufferSize);
  }

  private void setReceiveBufferSize(Socket sock) {
    setReceiveBufferSize(sock, this.owner.getConduit().tcpBufferSize);
  }

  private void setSendBufferSize(Socket sock, int requestedSize) {
    setSocketBufferSize(sock, true, requestedSize);
  }

  private void setReceiveBufferSize(Socket sock, int requestedSize) {
    setSocketBufferSize(sock, false, requestedSize);
  }

  public int getReceiveBufferSize() {
    return recvBufferSize;
  }

  private void setSocketBufferSize(Socket sock, boolean send, int requestedSize) {
    setSocketBufferSize(sock, send, requestedSize, false);
  }

  private void setSocketBufferSize(Socket sock, boolean send, int requestedSize,
      boolean alreadySetInSocket) {
    if (requestedSize > 0) {
      try {
        int currentSize = send ? sock.getSendBufferSize() : sock.getReceiveBufferSize();
        if (currentSize == requestedSize) {
          if (send) {
            this.sendBufferSize = currentSize;
          }
          return;
        }
        if (!alreadySetInSocket) {
          if (send) {
            sock.setSendBufferSize(requestedSize);
          } else {
            sock.setReceiveBufferSize(requestedSize);
          }
        } else {
        }
      } catch (SocketException ignore) {
      }
      try {
        int actualSize = send ? sock.getSendBufferSize() : sock.getReceiveBufferSize();
        if (send) {
          this.sendBufferSize = actualSize;
        } else {
          this.recvBufferSize = actualSize;
        }
        if (actualSize < requestedSize) {
          logger.info(LocalizedMessage.create(
              LocalizedStrings.Connection_SOCKET_0_IS_1_INSTEAD_OF_THE_REQUESTED_2,
              new Object[] {(send ? "send buffer size" : "receive buffer size"),
                  Integer.valueOf(actualSize), Integer.valueOf(requestedSize)}));
        } else if (actualSize > requestedSize) {
          if (logger.isTraceEnabled()) {
            logger.trace("Socket {} buffer size is {} instead of the requested {}",
                (send ? "send" : "receive"), actualSize, requestedSize);
          }
          // Remember the request size which is smaller.
          // This remembered value is used for allocating direct mem buffers.
          if (send) {
            this.sendBufferSize = requestedSize;
          } else {
            this.recvBufferSize = requestedSize;
          }
        }
      } catch (SocketException ignore) {
        if (send) {
          this.sendBufferSize = requestedSize;
        } else {
          this.recvBufferSize = requestedSize;
        }
      }
    }
  }

  /**
   * Returns the size of the send buffer on this connection's socket.
   */
  public int getSendBufferSize() {
    int result = this.sendBufferSize;
    if (result != -1) {
      return result;
    }
    try {
      result = getSocket().getSendBufferSize();
    } catch (SocketException ignore) {
      // just return a default
      result = this.owner.getConduit().tcpBufferSize;
    }
    this.sendBufferSize = result;
    return result;
  }

  /**
   * creates a connection that we accepted (it was initiated by an explicit connect being done on
   * the other side). We will only receive data on this socket; never send.
   */
  protected static Connection createReceiver(ConnectionTable table, Socket socket)
      throws IOException, ConnectionException {
    Connection connection = new Connection(table, socket);
    boolean readerStarted = false;
    try {
      connection.startReader(table);
      readerStarted = true;
    } finally {
      if (!readerStarted) {
        connection.closeForReconnect(
            LocalizedStrings.Connection_COULD_NOT_START_READER_THREAD.toLocalizedString());
      }
    }
    connection.waitForHandshake();
    connection.finishedConnecting = true;
    return connection;
  }

  /**
   * creates a connection that we accepted (it was initiated by an explicit connect being done on
   * the other side).
   */
  protected Connection(ConnectionTable t, Socket socket) throws IOException, ConnectionException {
    if (t == null) {
      throw new IllegalArgumentException(
          LocalizedStrings.Connection_NULL_CONNECTIONTABLE.toLocalizedString());
    }
    this.conduit = t.getConduit();
    this.isReceiver = true;
    this.owner = t;
    this.socket = socket;
    this.conduitIdStr = owner.getConduit().getSocketId().toString();
    this.handshakeRead = false;
    this.handshakeCancelled = false;
    this.connected = true;

    try {
      socket.setTcpNoDelay(true);
      socket.setKeepAlive(true);
      setSendBufferSize(socket, SMALL_BUFFER_SIZE);
      setReceiveBufferSize(socket);
    } catch (SocketException e) {
      // unable to get the settings we want. Don't log an error because it will
      // likely happen a lot
    }
    if (!useNIO()) {
      try {
        // this.output = new BufferedOutputStream(socket.getOutputStream(), SMALL_BUFFER_SIZE);
        this.output = socket.getOutputStream();
      } catch (IOException io) {
        logger.fatal(LocalizedMessage
            .create(LocalizedStrings.Connection_UNABLE_TO_GET_P2P_CONNECTION_STREAMS), io);
        t.getSocketCloser().asyncClose(socket, this.remoteAddr.toString(), null);
        throw io;
      }
    }
  }

  protected void initReceiver() {
    this.startReader(owner);
    this.waitForHandshake();
    this.finishedConnecting = true;
  }

  void setIdleTimeoutTask(SystemTimerTask task) {
    this.idleTask = task;
  }


  /**
   * Returns true if an idle connection was detected.
   */
  public boolean checkForIdleTimeout() {
    if (isSocketClosed()) {
      return true;
    }
    if (isSocketInUse() || (this.sharedResource && !this.preserveOrder)) { // shared/unordered
                                                                           // connections are used
                                                                           // for failure-detection
                                                                           // and are not subject to
                                                                           // idle-timeout
      return false;
    }
    boolean isIdle = !this.accessed;
    this.accessed = false;
    if (isIdle) {
      this.timedOut = true;
      this.owner.getConduit().getStats().incLostLease();
      if (logger.isDebugEnabled()) {
        logger.debug("Closing idle connection {} shared={} ordered={}", this, this.sharedResource,
            this.preserveOrder);
      }
      try {
        // Instead of calling requestClose
        // we call closeForReconnect.
        // We don't want this timeout close to close
        // any other connections. The problem with
        // requestClose has removeEndpoint set to true
        // which will close an receivers we have if this
        // connection is a shared one.
        closeForReconnect(
            LocalizedStrings.Connection_IDLE_CONNECTION_TIMED_OUT.toLocalizedString());
      } catch (Exception ignore) {
      }
    }
    return isIdle;
  }

  private static byte[] okHandshakeBytes;
  private static ByteBuffer okHandshakeBuf;
  static {
    int msglen = 1; // one byte for reply code
    byte[] bytes = new byte[MSG_HEADER_BYTES + msglen];
    msglen = calcHdrSize(msglen);
    bytes[MSG_HEADER_SIZE_OFFSET] = (byte) ((msglen / 0x1000000) & 0xff);
    bytes[MSG_HEADER_SIZE_OFFSET + 1] = (byte) ((msglen / 0x10000) & 0xff);
    bytes[MSG_HEADER_SIZE_OFFSET + 2] = (byte) ((msglen / 0x100) & 0xff);
    bytes[MSG_HEADER_SIZE_OFFSET + 3] = (byte) (msglen & 0xff);
    bytes[MSG_HEADER_TYPE_OFFSET] = (byte) NORMAL_MSG_TYPE; // message type
    bytes[MSG_HEADER_ID_OFFSET] = (byte) ((MsgIdGenerator.NO_MSG_ID >> 8) & 0xff);
    bytes[MSG_HEADER_ID_OFFSET + 1] = (byte) (MsgIdGenerator.NO_MSG_ID & 0xff);
    bytes[MSG_HEADER_BYTES] = REPLY_CODE_OK;
    int allocSize = bytes.length;
    ByteBuffer bb;
    if (TCPConduit.useDirectBuffers) {
      bb = ByteBuffer.allocateDirect(allocSize);
    } else {
      bb = ByteBuffer.allocate(allocSize);
    }
    bb.put(bytes);
    okHandshakeBuf = bb;
    okHandshakeBytes = bytes;
  }

  /**
   * maximum message buffer size
   */
  public static final int MAX_MSG_SIZE = 0x00ffffff;

  public static int calcHdrSize(int byteSize) {
    if (byteSize > MAX_MSG_SIZE) {
      throw new IllegalStateException(LocalizedStrings.Connection_TCP_MESSAGE_EXCEEDED_MAX_SIZE_OF_0
          .toLocalizedString(Integer.valueOf(MAX_MSG_SIZE)));
    }
    int hdrSize = byteSize;
    hdrSize |= (HANDSHAKE_VERSION << 24);
    return hdrSize;
  }

  public static int calcMsgByteSize(int hdrSize) {
    return hdrSize & MAX_MSG_SIZE;
  }

  public static byte calcHdrVersion(int hdrSize) throws IOException {
    byte ver = (byte) (hdrSize >> 24);
    if (ver != HANDSHAKE_VERSION) {
      throw new IOException(
          LocalizedStrings.Connection_DETECTED_WRONG_VERSION_OF_GEMFIRE_PRODUCT_DURING_HANDSHAKE_EXPECTED_0_BUT_FOUND_1
              .toLocalizedString(new Object[] {new Byte(HANDSHAKE_VERSION), new Byte(ver)}));
    }
    return ver;
  }

  private void sendOKHandshakeReply() throws IOException, ConnectionException {
    byte[] my_okHandshakeBytes = null;
    ByteBuffer my_okHandshakeBuf = null;
    if (this.isReceiver) {
      DistributionConfig cfg = owner.getConduit().config;
      ByteBuffer bb;
      if (useNIO() && TCPConduit.useDirectBuffers) {
        bb = ByteBuffer.allocateDirect(128);
      } else {
        bb = ByteBuffer.allocate(128);
      }
      bb.putInt(0); // reserve first 4 bytes for packet length
      bb.put((byte) NORMAL_MSG_TYPE);
      bb.putShort(MsgIdGenerator.NO_MSG_ID);
      bb.put(REPLY_CODE_OK_WITH_ASYNC_INFO);
      bb.putInt(cfg.getAsyncDistributionTimeout());
      bb.putInt(cfg.getAsyncQueueTimeout());
      bb.putInt(cfg.getAsyncMaxQueueSize());
      // write own product version
      Version.writeOrdinal(bb, Version.CURRENT.ordinal(), true);
      // now set the msg length into position 0
      bb.putInt(0, calcHdrSize(bb.position() - MSG_HEADER_BYTES));
      if (useNIO()) {
        my_okHandshakeBuf = bb;
        bb.flip();
      } else {
        my_okHandshakeBytes = new byte[bb.position()];
        bb.flip();
        bb.get(my_okHandshakeBytes);
      }
    } else {
      my_okHandshakeBuf = okHandshakeBuf;
      my_okHandshakeBytes = okHandshakeBytes;
    }
    if (useNIO()) {
      assert my_okHandshakeBuf != null;
      synchronized (my_okHandshakeBuf) {
        my_okHandshakeBuf.position(0);
        nioWriteFully(getSocket().getChannel(), my_okHandshakeBuf, false, null);
      }
    } else {
      synchronized (outLock) {
        assert my_okHandshakeBytes != null;
        this.output.write(my_okHandshakeBytes, 0, my_okHandshakeBytes.length);
        this.output.flush();
      }
    }
  }

  private static final int HANDSHAKE_TIMEOUT_MS =
      Integer.getInteger("p2p.handshakeTimeoutMs", 59000).intValue();
  // private static final byte HANDSHAKE_VERSION = 1; // 501
  // public static final byte HANDSHAKE_VERSION = 2; // cbb5x_PerfScale
  // public static final byte HANDSHAKE_VERSION = 3; // durable_client
  // public static final byte HANDSHAKE_VERSION = 4; // dataSerialMay19
  // public static final byte HANDSHAKE_VERSION = 5; // split-brain bits
  // public static final byte HANDSHAKE_VERSION = 6; // direct ack changes
  // NOTICE: handshake_version should not be changed anymore. Use the gemfire
  // version transmitted with the handshake bits and handle old handshakes
  // based on that
  public static final byte HANDSHAKE_VERSION = 7; // product version exchange during handshake

  /**
   * @throws ConnectionException if the conduit has stopped
   */
  private void waitForHandshake() throws ConnectionException {
    boolean needToClose = false;
    String reason = null;
    try {
      synchronized (this.handshakeSync) {
        if (!this.handshakeRead && !this.handshakeCancelled) {
          boolean success = false;
          reason = LocalizedStrings.Connection_UNKNOWN.toLocalizedString();
          boolean interrupted = Thread.interrupted();
          try {
            final long endTime = System.currentTimeMillis() + HANDSHAKE_TIMEOUT_MS;
            long msToWait = HANDSHAKE_TIMEOUT_MS;
            while (!this.handshakeRead && !this.handshakeCancelled && msToWait > 0) {
              this.handshakeSync.wait(msToWait); // spurious wakeup ok
              if (!this.handshakeRead && !this.handshakeCancelled) {
                msToWait = endTime - System.currentTimeMillis();
              }
            }
            if (!this.handshakeRead && !this.handshakeCancelled) {
              reason = LocalizedStrings.Connection_HANDSHAKE_TIMED_OUT.toLocalizedString();
              String peerName;
              if (this.remoteAddr != null) {
                peerName = this.remoteAddr.toString();
                // late in the life of jdk 1.7 we started seeing connections accepted
                // when accept() was not even being called. This started causing timeouts
                // to occur in the handshake threads instead of causing failures in
                // connection-formation. So, we need to initiate suspect processing here
                owner.getDM().getMembershipManager().suspectMember(this.remoteAddr,
                    LocalizedStrings.Connection_CONNECTION_HANDSHAKE_WITH_0_TIMED_OUT_AFTER_WAITING_1_MILLISECONDS
                        .toLocalizedString(
                            new Object[] {peerName, Integer.valueOf(HANDSHAKE_TIMEOUT_MS)}));
              } else {
                peerName = "socket " + this.socket.getRemoteSocketAddress().toString() + ":"
                    + this.socket.getPort();
              }
              throw new ConnectionException(
                  LocalizedStrings.Connection_CONNECTION_HANDSHAKE_WITH_0_TIMED_OUT_AFTER_WAITING_1_MILLISECONDS
                      .toLocalizedString(
                          new Object[] {peerName, Integer.valueOf(HANDSHAKE_TIMEOUT_MS)}));
            } else {
              success = this.handshakeRead;
            }
          } catch (InterruptedException ex) {
            interrupted = true;
            this.owner.getConduit().getCancelCriterion().checkCancelInProgress(ex);
            reason = LocalizedStrings.Connection_INTERRUPTED.toLocalizedString();
          } finally {
            if (interrupted) {
              Thread.currentThread().interrupt();
            }
            if (success) {
              if (this.isReceiver) {
                needToClose =
                    !owner.getConduit().getMembershipManager().addSurpriseMember(this.remoteAddr);
                if (needToClose) {
                  reason = "this member is shunned";
                }
              }
            } else {
              needToClose = true; // for bug 42159
            }
          }
        } // !handshakeRead
      } // synchronized

    } finally {
      if (needToClose) {
        // moved this call outside of the sync for bug 42159
        try {
          requestClose(reason); // fix for bug 31546
        } catch (Exception ignore) {
        }
      }
    }
  }

  private void notifyHandshakeWaiter(boolean success) {
    synchronized (this.handshakeSync) {
      if (success) {
        this.handshakeRead = true;
      } else {
        this.handshakeCancelled = true;
      }
      this.handshakeSync.notify();
    }
  }

  private final AtomicBoolean asyncCloseCalled = new AtomicBoolean();

  /**
   * asynchronously close this connection
   *
   * @param beingSick test hook to simulate sickness in communications & membership
   */
  private void asyncClose(boolean beingSick) {
    // note: remoteAddr may be null if this is a receiver that hasn't finished its handshake

    // we do the close in a background thread because the operation may hang if
    // there is a problem with the network. See bug #46659

    // if simulating sickness, sockets must be closed in-line so that tests know
    // that the vm is sick when the beSick operation completes
    if (beingSick) {
      prepareForAsyncClose();
    } else {
      if (this.asyncCloseCalled.compareAndSet(false, true)) {
        Socket s = this.socket;
        if (s != null && !s.isClosed()) {
          prepareForAsyncClose();
          this.owner.getSocketCloser().asyncClose(s, String.valueOf(this.remoteAddr), null);
        }
      }
    }
  }

  private void prepareForAsyncClose() {
    synchronized (stateLock) {
      if (readerThread != null && isRunning && !readerShuttingDown
          && (connectionState == STATE_READING || connectionState == STATE_READING_ACK)) {
        readerThread.interrupt();
      }
    }
  }

  private static final int CONNECT_HANDSHAKE_SIZE = 4096;

  /**
   * waits until we've joined the distributed system before returning
   */
  private void waitForAddressCompletion() {
    InternalDistributedMember myAddr = this.owner.getConduit().getMemberId();
    synchronized (myAddr) {
      while ((!owner.getConduit().getCancelCriterion().isCancelInProgress())
          && myAddr.getInetAddress() == null && myAddr.getVmViewId() < 0) {
        try {
          myAddr.wait(100); // spurious wakeup ok
        } catch (InterruptedException ie) {
          Thread.currentThread().interrupt();
          this.owner.getConduit().getCancelCriterion().checkCancelInProgress(ie);
        }
      }
      Assert.assertTrue(myAddr.getDirectChannelPort() == this.owner.getConduit().getPort());
    }
  }

  private void handshakeNio() throws IOException {
    waitForAddressCompletion();

    InternalDistributedMember myAddr = this.owner.getConduit().getMemberId();
    final MsgOutputStream connectHandshake = new MsgOutputStream(CONNECT_HANDSHAKE_SIZE);
    /*
     * Note a byte of zero is always written because old products serialized a member id with always
     * sends an ip address. My reading of the ip-address specs indicated that the first byte of a
     * valid address would never be 0.
     */
    connectHandshake.writeByte(0);
    connectHandshake.writeByte(HANDSHAKE_VERSION);
    // NOTE: if you add or remove code in this section bump HANDSHAKE_VERSION
    InternalDataSerializer.invokeToData(myAddr, connectHandshake);
    connectHandshake.writeBoolean(this.sharedResource);
    connectHandshake.writeBoolean(this.preserveOrder);
    connectHandshake.writeLong(this.uniqueId);
    // write the product version ordinal
    Version.CURRENT.writeOrdinal(connectHandshake, true);
    connectHandshake.writeInt(dominoCount.get() + 1);
    // this writes the sending member + thread name that is stored in senderName
    // on the receiver to show the cause of reader thread creation
    // if (dominoCount.get() > 0) {
    // connectHandshake.writeUTF(Thread.currentThread().getName());
    // } else {
    // String name = owner.getDM().getConfig().getName();
    // if (name == null) {
    // name = "pid="+OSProcess.getId();
    // }
    // connectHandshake.writeUTF("["+name+"] "+Thread.currentThread().getName());
    // }
    connectHandshake.setMessageHeader(NORMAL_MSG_TYPE, ClusterDistributionManager.STANDARD_EXECUTOR,
        MsgIdGenerator.NO_MSG_ID);
    nioWriteFully(getSocket().getChannel(), connectHandshake.getContentBuffer(), false, null);
  }

  private void handshakeStream() throws IOException {
    waitForAddressCompletion();

    this.output = getSocket().getOutputStream();
    ByteArrayOutputStream baos = new ByteArrayOutputStream(CONNECT_HANDSHAKE_SIZE);
    DataOutputStream os = new DataOutputStream(baos);
    InternalDistributedMember myAddr = owner.getConduit().getMemberId();
    os.writeByte(0);
    os.writeByte(HANDSHAKE_VERSION);
    // NOTE: if you add or remove code in this section bump HANDSHAKE_VERSION
    InternalDataSerializer.invokeToData(myAddr, os);
    os.writeBoolean(this.sharedResource);
    os.writeBoolean(this.preserveOrder);
    os.writeLong(this.uniqueId);
    Version.CURRENT.writeOrdinal(os, true);
    os.writeInt(dominoCount.get() + 1);
    os.flush();

    byte[] msg = baos.toByteArray();
    int len = calcHdrSize(msg.length);
    byte[] lenbytes = new byte[MSG_HEADER_BYTES];
    lenbytes[MSG_HEADER_SIZE_OFFSET] = (byte) ((len / 0x1000000) & 0xff);
    lenbytes[MSG_HEADER_SIZE_OFFSET + 1] = (byte) ((len / 0x10000) & 0xff);
    lenbytes[MSG_HEADER_SIZE_OFFSET + 2] = (byte) ((len / 0x100) & 0xff);
    lenbytes[MSG_HEADER_SIZE_OFFSET + 3] = (byte) (len & 0xff);
    lenbytes[MSG_HEADER_TYPE_OFFSET] = (byte) NORMAL_MSG_TYPE;
    lenbytes[MSG_HEADER_ID_OFFSET] = (byte) ((MsgIdGenerator.NO_MSG_ID >> 8) & 0xff);
    lenbytes[MSG_HEADER_ID_OFFSET + 1] = (byte) (MsgIdGenerator.NO_MSG_ID & 0xff);
    synchronized (outLock) {
      this.output.write(lenbytes, 0, lenbytes.length);
      this.output.write(msg, 0, msg.length);
      this.output.flush();
    }
  }

  /**
   *
   * @throws IOException if handshake fails
   */
  private void attemptHandshake(ConnectionTable connTable) throws IOException {
    // send HANDSHAKE
    // send this server's port. It's expected on the other side
    if (useNIO()) {
      handshakeNio();
    } else {
      handshakeStream();
    }

    startReader(connTable); // this reader only reads the handshake and then exits
    waitForHandshake(); // waiting for reply
  }

  /** time between connection attempts */
  private static final int RECONNECT_WAIT_TIME = Integer
      .getInteger(DistributionConfig.GEMFIRE_PREFIX + "RECONNECT_WAIT_TIME", 2000).intValue();

  /**
   * creates a new connection to a remote server. We are initiating this connection; the other side
   * must accept us We will almost always send messages; small acks are received.
   */
  protected static Connection createSender(final MembershipManager mgr, final ConnectionTable t,
      final boolean preserveOrder, final DistributedMember remoteAddr, final boolean sharedResource,
      final long startTime, final long ackTimeout, final long ackSATimeout)
      throws IOException, DistributedSystemDisconnectedException {
    boolean warningPrinted = false;
    boolean success = false;
    boolean firstTime = true;
    Connection conn = null;
    // keep trying. Note that this may be executing during the shutdown window
    // where a cancel criterion has not been established, but threads are being
    // interrupted. In this case we must allow the connection to succeed even
    // though subsequent messaging using the socket may fail
    boolean interrupted = Thread.interrupted();
    boolean severeAlertIssued = false;
    boolean suspected = false;
    long reconnectWaitTime = RECONNECT_WAIT_TIME;
    boolean connectionErrorLogged = false;
    try {
      while (!success) { // keep trying
        // Quit if DM has stopped distribution
        t.getConduit().getCancelCriterion().checkCancelInProgress(null);
        long now = System.currentTimeMillis();
        if (!severeAlertIssued && ackSATimeout > 0 && startTime + ackTimeout < now) {
          if (startTime + ackTimeout + ackSATimeout < now) {
            if (remoteAddr != null) {
              logger.fatal(LocalizedMessage.create(
                  LocalizedStrings.Connection_UNABLE_TO_FORM_A_TCPIP_CONNECTION_TO_0_IN_OVER_1_SECONDS,
                  new Object[] {remoteAddr, (ackSATimeout + ackTimeout) / 1000}));
            }
            severeAlertIssued = true;
          } else if (!suspected) {
            if (remoteAddr != null) {
              logger.warn(LocalizedMessage.create(
                  LocalizedStrings.Connection_UNABLE_TO_FORM_A_TCPIP_CONNECTION_TO_0_IN_OVER_1_SECONDS,
                  new Object[] {remoteAddr, (ackTimeout) / 1000}));
            }
            mgr.suspectMember(remoteAddr,
                LocalizedStrings.Connection_UNABLE_TO_FORM_A_TCPIP_CONNECTION_IN_A_REASONABLE_AMOUNT_OF_TIME
                    .toLocalizedString());
            suspected = true;
          }
          reconnectWaitTime =
              Math.min(RECONNECT_WAIT_TIME, ackSATimeout - (now - startTime - ackTimeout));
          if (reconnectWaitTime <= 0) {
            reconnectWaitTime = RECONNECT_WAIT_TIME;
          }
        } else if (!suspected && (startTime > 0) && (ackTimeout > 0)
            && (startTime + ackTimeout < now)) {
          mgr.suspectMember(remoteAddr,
              LocalizedStrings.Connection_UNABLE_TO_FORM_A_TCPIP_CONNECTION_IN_A_REASONABLE_AMOUNT_OF_TIME
                  .toLocalizedString());
          suspected = true;
        }
        if (firstTime) {
          firstTime = false;
          if (!mgr.memberExists(remoteAddr) || mgr.isShunned(remoteAddr)
              || mgr.shutdownInProgress()) {
            throw new IOException("Member " + remoteAddr + " left the system");
          }
        } else {
          // if we're sending an alert and can't connect, bail out. A sick
          // alert listener should not prevent cache operations from continuing
          if (AlertAppender.isThreadAlerting()) {
            // do not change the text of this exception - it is looked for in exception handlers
            throw new IOException("Cannot form connection to alert listener " + remoteAddr);
          }

          // Wait briefly...
          interrupted = Thread.interrupted() || interrupted;
          try {
            Thread.sleep(reconnectWaitTime);
          } catch (InterruptedException ie) {
            interrupted = true;
            t.getConduit().getCancelCriterion().checkCancelInProgress(ie);
          }
          t.getConduit().getCancelCriterion().checkCancelInProgress(null);
          if (giveUpOnMember(mgr, remoteAddr)) {
            throw new IOException(
                LocalizedStrings.Connection_MEMBER_LEFT_THE_GROUP.toLocalizedString(remoteAddr));
          }
          if (!warningPrinted) {
            warningPrinted = true;
            logger.warn(LocalizedMessage.create(
                LocalizedStrings.Connection_CONNECTION_ATTEMPTING_RECONNECT_TO_PEER__0,
                remoteAddr));
          }
          t.getConduit().getStats().incReconnectAttempts();
        }
        // create connection
        try {
          conn = null;
          conn = new Connection(t, preserveOrder, remoteAddr, sharedResource);
        } catch (javax.net.ssl.SSLHandshakeException se) {
          // no need to retry if certificates were rejected
          throw se;
        } catch (IOException ioe) {
          // Only give up if the member leaves the view.
          if (giveUpOnMember(mgr, remoteAddr)) {
            throw ioe;
          }
          t.getConduit().getCancelCriterion().checkCancelInProgress(null);
          if ("Too many open files".equals(ioe.getMessage())) {
            t.fileDescriptorsExhausted();
          } else if (!connectionErrorLogged) {
            connectionErrorLogged = true; // otherwise change to use 100ms intervals causes a lot of
                                          // these
            logger.info(LocalizedMessage.create(
                LocalizedStrings.Connection_CONNECTION_FAILED_TO_CONNECT_TO_PEER_0_BECAUSE_1,
                new Object[] {sharedResource, preserveOrder, remoteAddr, ioe}));
          }
        } // IOException
        finally {
          if (conn == null) {
            t.getConduit().getStats().incFailedConnect();
          }
        }
        if (conn != null) {
          // handshake
          try {
            conn.attemptHandshake(t);
            if (conn.isSocketClosed()) {
              // something went wrong while reading the handshake
              // and the socket was closed or this guy sent us a
              // ShutdownMessage
              if (giveUpOnMember(mgr, remoteAddr)) {
                throw new IOException(LocalizedStrings.Connection_MEMBER_LEFT_THE_GROUP
                    .toLocalizedString(remoteAddr));
              }
              t.getConduit().getCancelCriterion().checkCancelInProgress(null);
              // no success but no need to log; just retry
            } else {
              success = true;
            }
          } catch (ConnectionException e) {
            if (giveUpOnMember(mgr, remoteAddr)) {
              IOException ioe =
                  new IOException(LocalizedStrings.Connection_HANDSHAKE_FAILED.toLocalizedString());
              ioe.initCause(e);
              throw ioe;
            }
            t.getConduit().getCancelCriterion().checkCancelInProgress(null);
            logger.info(LocalizedMessage.create(
                LocalizedStrings.Connection_CONNECTION_HANDSHAKE_FAILED_TO_CONNECT_TO_PEER_0_BECAUSE_1,
                new Object[] {sharedResource, preserveOrder, remoteAddr, e}));
          } catch (IOException e) {
            if (giveUpOnMember(mgr, remoteAddr)) {
              throw e;
            }
            t.getConduit().getCancelCriterion().checkCancelInProgress(null);
            logger.info(LocalizedMessage.create(
                LocalizedStrings.Connection_CONNECTION_HANDSHAKE_FAILED_TO_CONNECT_TO_PEER_0_BECAUSE_1,
                new Object[] {sharedResource, preserveOrder, remoteAddr, e}));
            if (!sharedResource && "Too many open files".equals(e.getMessage())) {
              t.fileDescriptorsExhausted();
            }
          } finally {
            if (!success) {
              try {
                conn.requestClose(LocalizedStrings.Connection_FAILED_HANDSHAKE.toLocalizedString());
              } catch (Exception ignore) {
              }
              conn = null;
            }
          }
        }
      } // while
      if (warningPrinted) {
        logger.info(LocalizedMessage.create(
            LocalizedStrings.Connection_0_SUCCESSFULLY_REESTABLISHED_CONNECTION_TO_PEER_1,
            new Object[] {mgr.getLocalMember(), remoteAddr}));
      }
    } finally {
      try {
        if (!success) {
          if (conn != null) {
            conn.requestClose(LocalizedStrings.Connection_FAILED_CONSTRUCTION.toLocalizedString());
            conn = null;
          }
        }
      } finally {
        if (interrupted) {
          Thread.currentThread().interrupt();
        }
      }
    }
    // Assert.assertTrue(conn != null);
    if (conn == null) {
      throw new ConnectionException(
          LocalizedStrings.Connection_CONNECTION_FAILED_CONSTRUCTION_FOR_PEER_0
              .toLocalizedString(remoteAddr));
    }
    if (preserveOrder && BATCH_SENDS) {
      conn.createBatchSendBuffer();
    }
    conn.finishedConnecting = true;
    return conn;
  }

  private static boolean giveUpOnMember(MembershipManager mgr, DistributedMember remoteAddr) {
    return !mgr.memberExists(remoteAddr) || mgr.isShunned(remoteAddr) || mgr.shutdownInProgress();
  }

  private void setRemoteAddr(DistributedMember m) {
    this.remoteAddr = this.owner.getDM().getCanonicalId(m);
    MembershipManager mgr = this.conduit.getMembershipManager();
    mgr.addSurpriseMember(m);
  }

  /**
   * creates a new connection to a remote server. We are initiating this connection; the other side
   * must accept us We will almost always send messages; small acks are received.
   */
  private Connection(ConnectionTable t, boolean preserveOrder, DistributedMember remoteID,
      boolean sharedResource) throws IOException, DistributedSystemDisconnectedException {

    // initialize a socket upfront. So that the
    InternalDistributedMember remoteAddr = (InternalDistributedMember) remoteID;
    if (t == null) {
      throw new IllegalArgumentException(
          LocalizedStrings.Connection_CONNECTIONTABLE_IS_NULL.toLocalizedString());
    }
    this.conduit = t.getConduit();
    this.isReceiver = false;
    this.owner = t;
    this.sharedResource = sharedResource;
    this.preserveOrder = preserveOrder;
    setRemoteAddr(remoteAddr);
    this.conduitIdStr = this.owner.getConduit().getSocketId().toString();
    this.handshakeRead = false;
    this.handshakeCancelled = false;
    this.connected = true;

    this.uniqueId = idCounter.getAndIncrement();

    // connect to listening socket

    InetSocketAddress addr =
        new InetSocketAddress(remoteAddr.getInetAddress(), remoteAddr.getDirectChannelPort());
    if (useNIO()) {
      SocketChannel channel = SocketChannel.open();
      this.owner.addConnectingSocket(channel.socket(), addr.getAddress());
      try {
        channel.socket().setTcpNoDelay(true);

        channel.socket().setKeepAlive(SocketCreator.ENABLE_TCP_KEEP_ALIVE);

        /*
         * If conserve-sockets is false, the socket can be used for receiving responses, so set the
         * receive buffer accordingly.
         */
        if (!sharedResource) {
          setReceiveBufferSize(channel.socket(), this.owner.getConduit().tcpBufferSize);
        } else {
          setReceiveBufferSize(channel.socket(), SMALL_BUFFER_SIZE); // make small since only
                                                                     // receive ack messages
        }
        setSendBufferSize(channel.socket());
        channel.configureBlocking(true);

        int connectTime = getP2PConnectTimeout();

        try {
          channel.socket().connect(addr, connectTime);
        } catch (NullPointerException e) {
          // bug #45044 - jdk 1.7 sometimes throws an NPE here
          ConnectException c = new ConnectException("Encountered bug #45044 - retrying");
          c.initCause(e);
          // prevent a hot loop by sleeping a little bit
          try {
            Thread.sleep(1000);
          } catch (InterruptedException ie) {
            Thread.currentThread().interrupt();
          }
          throw c;
        } catch (CancelledKeyException | ClosedSelectorException e) {
          // bug #44469: for some reason NIO throws this runtime exception
          // instead of an IOException on timeouts
          ConnectException c = new ConnectException(
              LocalizedStrings.Connection_ATTEMPT_TO_CONNECT_TIMED_OUT_AFTER_0_MILLISECONDS
                  .toLocalizedString(new Object[] {connectTime}));
          c.initCause(e);
          throw c;
        }
      } finally {
        this.owner.removeConnectingSocket(channel.socket());
      }
      this.socket = channel.socket();
    } else {
      if (TCPConduit.useSSL) {
        int socketBufferSize =
            sharedResource ? SMALL_BUFFER_SIZE : this.owner.getConduit().tcpBufferSize;
        this.socket = owner.getConduit().getSocketCreator().connectForServer(
            remoteAddr.getInetAddress(), remoteAddr.getDirectChannelPort(), socketBufferSize);
        // Set the receive buffer size local fields. It has already been set in the socket.
        setSocketBufferSize(this.socket, false, socketBufferSize, true);
        setSendBufferSize(this.socket);
      } else {
        Socket s = new Socket();
        this.socket = s;
        s.setTcpNoDelay(true);
        s.setKeepAlive(SocketCreator.ENABLE_TCP_KEEP_ALIVE);
        setReceiveBufferSize(s, SMALL_BUFFER_SIZE);
        setSendBufferSize(s);
        s.connect(addr, 0);
      }
    }
    if (logger.isDebugEnabled()) {
      logger.debug("Connection: connected to {} with stub {}", remoteAddr, addr);
    }
    try {
      getSocket().setTcpNoDelay(true);
    } catch (SocketException e) {
    }
  }

  /**
   * Batch sends currently should not be turned on because: 1. They will be used for all sends
   * (instead of just no-ack) and thus will break messages that wait for a response (or kill perf).
   * 2. The buffer is not properly flushed and closed on shutdown. The code attempts to do this but
   * must not be doing it correctly.
   */
  private static final boolean BATCH_SENDS = Boolean.getBoolean("p2p.batchSends");
  private static final int BATCH_BUFFER_SIZE =
      Integer.getInteger("p2p.batchBufferSize", 1024 * 1024).intValue();
  private static final int BATCH_FLUSH_MS = Integer.getInteger("p2p.batchFlushTime", 50).intValue();
  private Object batchLock;
  private ByteBuffer fillBatchBuffer;
  private ByteBuffer sendBatchBuffer;
  private BatchBufferFlusher batchFlusher;

  private void createBatchSendBuffer() {
    // batch send buffer isn't needed if old-io is being used
    if (!this.useNIO) {
      return;
    }
    this.batchLock = new Object();
    if (TCPConduit.useDirectBuffers) {
      this.fillBatchBuffer = ByteBuffer.allocateDirect(BATCH_BUFFER_SIZE);
      this.sendBatchBuffer = ByteBuffer.allocateDirect(BATCH_BUFFER_SIZE);
    } else {
      this.fillBatchBuffer = ByteBuffer.allocate(BATCH_BUFFER_SIZE);
      this.sendBatchBuffer = ByteBuffer.allocate(BATCH_BUFFER_SIZE);
    }
    this.batchFlusher = new BatchBufferFlusher();
    this.batchFlusher.start();
  }

  public void cleanUpOnIdleTaskCancel() {
    // Make sure receivers are removed from the connection table, this should always be a noop, but
    // is done here as a failsafe.
    if (isReceiver) {
      owner.removeReceiver(this);
    }
  }

  private class BatchBufferFlusher extends Thread {
    private volatile boolean flushNeeded = false;
    private volatile boolean timeToStop = false;
    private DMStats stats;


    public BatchBufferFlusher() {
      setDaemon(true);
      this.stats = owner.getConduit().getStats();
    }

    /**
     * Called when a message writer needs the current fillBatchBuffer flushed
     */
    public void flushBuffer(ByteBuffer bb) {
      final long start = DistributionStats.getStatTime();
      try {
        synchronized (this) {
          synchronized (batchLock) {
            if (bb != fillBatchBuffer) {
              // it must have already been flushed. So just return
              // and use the new fillBatchBuffer
              return;
            }
          }
          this.flushNeeded = true;
          this.notify();
        }
        synchronized (batchLock) {
          // Wait for the flusher thread
          while (bb == fillBatchBuffer) {
            Connection.this.owner.getConduit().getCancelCriterion().checkCancelInProgress(null);
            boolean interrupted = Thread.interrupted();
            try {
              batchLock.wait(); // spurious wakeup ok
            } catch (InterruptedException ex) {
              interrupted = true;
            } finally {
              if (interrupted) {
                Thread.currentThread().interrupt();
              }
            }
          } // while
        }
      } finally {
        owner.getConduit().getStats().incBatchWaitTime(start);
      }
    }

    public void close() {
      synchronized (this) {
        this.timeToStop = true;
        this.flushNeeded = true;
        this.notify();
      }
    }

    @Override
    public void run() {
      try {
        synchronized (this) {
          while (!timeToStop) {
            if (!this.flushNeeded && fillBatchBuffer.position() <= (BATCH_BUFFER_SIZE / 2)) {
              wait(BATCH_FLUSH_MS); // spurious wakeup ok
            }
            if (this.flushNeeded || fillBatchBuffer.position() > (BATCH_BUFFER_SIZE / 2)) {
              final long start = DistributionStats.getStatTime();
              synchronized (batchLock) {
                // This is the only block of code that will swap
                // the buffer references
                this.flushNeeded = false;
                ByteBuffer tmp = fillBatchBuffer;
                fillBatchBuffer = sendBatchBuffer;
                sendBatchBuffer = tmp;
                batchLock.notifyAll();
              }
              // We now own the sendBatchBuffer
              if (sendBatchBuffer.position() > 0) {
                final boolean origSocketInUse = socketInUse;
                socketInUse = true;
                try {
                  sendBatchBuffer.flip();
                  SocketChannel channel = getSocket().getChannel();
                  nioWriteFully(channel, sendBatchBuffer, false, null);
                  sendBatchBuffer.clear();
                } catch (IOException | ConnectionException ex) {
                  logger.fatal(LocalizedMessage.create(
                      LocalizedStrings.Connection_EXCEPTION_FLUSHING_BATCH_SEND_BUFFER_0, ex));
                  readerShuttingDown = true;
                  requestClose(LocalizedStrings.Connection_EXCEPTION_FLUSHING_BATCH_SEND_BUFFER_0
                      .toLocalizedString(ex));
                } finally {
                  accessed();
                  socketInUse = origSocketInUse;
                }
              }
              this.stats.incBatchFlushTime(start);
            }
          }
        }
      } catch (InterruptedException ex) {
        // time for this thread to shutdown
        // Thread.currentThread().interrupt();
      }
    }
  }

  private void closeBatchBuffer() {
    if (this.batchFlusher != null) {
      this.batchFlusher.close();
    }
  }

  /**
   * use to test message prep overhead (no socket write). WARNING: turning this on completely
   * disables distribution of batched sends
   */
  private static final boolean SOCKET_WRITE_DISABLED = Boolean.getBoolean("p2p.disableSocketWrite");

  private void batchSend(ByteBuffer src) throws IOException {
    if (SOCKET_WRITE_DISABLED) {
      return;
    }
    final long start = DistributionStats.getStatTime();
    try {
      ByteBuffer dst = null;
      Assert.assertTrue(src.remaining() <= BATCH_BUFFER_SIZE, "Message size(" + src.remaining()
          + ") exceeded BATCH_BUFFER_SIZE(" + BATCH_BUFFER_SIZE + ")");
      do {
        synchronized (this.batchLock) {
          dst = this.fillBatchBuffer;
          if (src.remaining() <= dst.remaining()) {
            final long copyStart = DistributionStats.getStatTime();
            dst.put(src);
            this.owner.getConduit().getStats().incBatchCopyTime(copyStart);
            return;
          }
        }
        // If we got this far then we do not have room in the current
        // buffer and need the flusher thread to flush before we can fill it
        this.batchFlusher.flushBuffer(dst);
      } while (true);
    } finally {
      this.owner.getConduit().getStats().incBatchSendTime(start);
    }
  }

  /**
   * Request that the manager close this connection, or close it forcibly if there is no manager.
   * Invoking this method ensures that the proper synchronization is done.
   */
  void requestClose(String reason) {
    close(reason, true, true, false, false);
  }

  boolean isClosing() {
    return this.closing.get();
  }

  void closePartialConnect(String reason, boolean beingSick) {
    close(reason, false, false, beingSick, false);
  }

  void closeForReconnect(String reason) {
    close(reason, true, false, false, false);
  }

  void closeOldConnection(String reason) {
    close(reason, true, true, false, true);
  }

  /**
   * Closes the connection.
   *
   * @see #requestClose
   */
  @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "TLW_TWO_LOCK_WAIT")
  private void close(String reason, boolean cleanupEndpoint, boolean p_removeEndpoint,
      boolean beingSick, boolean forceRemoval) {
    boolean removeEndpoint = p_removeEndpoint;
    // use getAndSet outside sync on this to fix 42330
    boolean onlyCleanup = this.closing.getAndSet(true);
    if (onlyCleanup && !forceRemoval) {
      return;
    }
    if (!onlyCleanup) {
      synchronized (this) {
        this.stopped = true;
        if (this.connected) {
          if (this.asyncQueuingInProgress && this.pusherThread != Thread.currentThread()) {
            // We don't need to do this if we are the pusher thread
            // and we have determined that we need to close the connection.
            // See bug 37601.
            synchronized (this.outgoingQueue) {
              // wait for the flusher to complete (it may timeout)
              while (this.asyncQueuingInProgress) {
                // Don't do this: causes closes to not get done in the event
                // of an orderly shutdown:
                // this.owner.getConduit().getCancelCriterion().checkCancelInProgress(null);
                boolean interrupted = Thread.interrupted();
                try {
                  this.outgoingQueue.wait(); // spurious wakeup ok
                } catch (InterruptedException ie) {
                  interrupted = true;
                  // this.owner.getConduit().getCancelCriterion().checkCancelInProgress(ie);
                } finally {
                  if (interrupted)
                    Thread.currentThread().interrupt();
                }
              } // while
            } // synchronized
          }
          this.connected = false;
          closeSenderSem();
          {
            final DMStats stats = this.owner.getConduit().getStats();
            if (this.finishedConnecting) {
              if (this.isReceiver) {
                stats.decReceivers();
              } else {
                stats.decSenders(this.sharedResource, this.preserveOrder);
              }
            }
          }
          if (logger.isDebugEnabled()) {
            logger.debug("Closing socket for {}", this);
          }
        } else if (!forceRemoval) {
          removeEndpoint = false;
        }
        // make sure our socket is closed
        asyncClose(false);
        nioLengthSet = false;
      } // synchronized

      // moved the call to notifyHandshakeWaiter out of the above
      // synchronized block to fix bug #42159
      // Make sure anyone waiting for a handshake stops waiting
      notifyHandshakeWaiter(false);
      // wait a bit for the our reader thread to exit
      // don't wait if we are the reader thread
      boolean isIBM = false;
      // if network partition detection is enabled or this is an admin vm
      // we can't wait for the reader thread when running in an IBM JRE. See
      // bug 41889
      if (this.conduit.config.getEnableNetworkPartitionDetection()
          || this.conduit.getMemberId().getVmKind() == ClusterDistributionManager.ADMIN_ONLY_DM_TYPE
          || this.conduit.getMemberId().getVmKind() == ClusterDistributionManager.LOCATOR_DM_TYPE) {
        isIBM = "IBM Corporation".equals(System.getProperty("java.vm.vendor"));
      }
      {
        // Now that readerThread is returned to a pool after we close
        // we need to be more careful not to join on a thread that belongs
        // to someone else.
        Thread readerThreadSnapshot = this.readerThread;
        if (!beingSick && readerThreadSnapshot != null && !isIBM && this.isRunning
            && !this.readerShuttingDown && readerThreadSnapshot != Thread.currentThread()) {
          try {
            readerThreadSnapshot.join(500);
            readerThreadSnapshot = this.readerThread;
            if (this.isRunning && !this.readerShuttingDown && readerThreadSnapshot != null
                && owner.getDM().getRootCause() == null) { // don't wait twice if there's a system
                                                           // failure
              readerThreadSnapshot.join(1500);
              if (this.isRunning) {
                logger.info(LocalizedMessage.create(
                    LocalizedStrings.Connection_TIMED_OUT_WAITING_FOR_READERTHREAD_ON_0_TO_FINISH,
                    this));
              }
            }
          } catch (IllegalThreadStateException ignore) {
            // ignored - thread already stopped
          } catch (InterruptedException ignore) {
            Thread.currentThread().interrupt();
            // but keep going, we're trying to close.
          }
        }
      }

      closeBatchBuffer();
      closeAllMsgDestreamers();
    }
    if (cleanupEndpoint) {
      if (this.isReceiver) {
        this.owner.removeReceiver(this);
      }
      if (removeEndpoint) {
        if (this.sharedResource) {
          if (!this.preserveOrder) {
            // only remove endpoint when shared unordered connection
            // is closed. This is part of the fix for bug 32980.
            if (!this.isReceiver) {
              // Only remove endpoint if sender.
              if (this.finishedConnecting) {
                // only remove endpoint if our constructor finished
                this.owner.removeEndpoint(this.remoteAddr, reason);
              }
            }
          } else {
            // noinspection ConstantConditions
            this.owner.removeSharedConnection(reason, this.remoteAddr, this.preserveOrder, this);
          }
        } else if (!this.isReceiver) {
          this.owner.removeThreadConnection(this.remoteAddr, this);
        }
      } else {
        // This code is ok to do even if the ConnectionTable
        // has never added this Connection to its maps since
        // the calls in this block use our identity to do the removes.
        if (this.sharedResource) {
          this.owner.removeSharedConnection(reason, this.remoteAddr, this.preserveOrder, this);
        } else if (!this.isReceiver) {
          this.owner.removeThreadConnection(this.remoteAddr, this);
        }
      }
    }

    // This cancels the idle timer task, but it also removes the tasks
    // reference to this connection, freeing up the connection (and it's buffers
    // for GC sooner.
    if (idleTask != null) {
      idleTask.cancel();
    }

    if (ackTimeoutTask != null) {
      ackTimeoutTask.cancel();
    }

  }

  /** starts a reader thread */
  private void startReader(ConnectionTable connTable) {
    Assert.assertTrue(!this.isRunning);
    stopped = false;
    this.isRunning = true;
    connTable.executeCommand(this);
  }


  /**
   * in order to read non-NIO socket-based messages we need to have a thread actively trying to grab
   * bytes out of the sockets input queue. This is that thread.
   */
  public void run() {
    this.readerThread = Thread.currentThread();
    this.readerThread.setName(p2pReaderName());
    ConnectionTable.threadWantsSharedResources();
    makeReaderThread(this.isReceiver);
    try {
      if (useNIO()) {
        runNioReader();
      } else {
        runOioReader();
      }
    } finally {
      // bug36060: do the socket close within a finally block
      if (logger.isDebugEnabled()) {
        logger.debug("Stopping {} for {}", p2pReaderName(), remoteAddr);
      }
      initiateSuspicionIfSharedUnordered();
      if (this.isReceiver) {
        if (!this.sharedResource) {
          this.conduit.getStats().incThreadOwnedReceivers(-1L, dominoCount.get());
        }
        asyncClose(false);
        this.owner.removeAndCloseThreadOwnedSockets();
      }
      ByteBuffer tmp = this.nioInputBuffer;
      if (tmp != null) {
        this.nioInputBuffer = null;
        final DMStats stats = this.owner.getConduit().getStats();
        Buffers.releaseReceiveBuffer(tmp, stats);
      }
      // make sure that if the reader thread exits we notify a thread waiting
      // for the handshake.
      // see bug 37524 for an example of listeners hung in waitForHandshake
      notifyHandshakeWaiter(false);
      this.readerThread.setName("unused p2p reader");
      synchronized (this.stateLock) {
        this.isRunning = false;
        this.readerThread = null;
      }
    } // finally
  }

  private String p2pReaderName() {
    StringBuilder sb = new StringBuilder(64);
    if (this.isReceiver) {
      sb.append("P2P message reader@");
    } else {
      sb.append("P2P handshake reader@");
    }
    sb.append(Integer.toHexString(System.identityHashCode(this)));
    if (!this.isReceiver) {
      sb.append('-').append(getUniqueId());
    }
    return sb.toString();
  }

  private void runNioReader() {
    // take a snapshot of uniqueId to detect reconnect attempts; see bug 37592
    SocketChannel channel = null;
    try {
      channel = getSocket().getChannel();
      channel.configureBlocking(true);
    } catch (ClosedChannelException e) {
      // bug 37693: the channel was asynchronously closed. Our work
      // is done.
      try {
        requestClose(
            LocalizedStrings.Connection_RUNNIOREADER_CAUGHT_CLOSED_CHANNEL.toLocalizedString());
      } catch (Exception ignore) {
      }
      return; // exit loop and thread
    } catch (IOException ex) {
      if (stopped || owner.getConduit().getCancelCriterion().isCancelInProgress()) {
        try {
          requestClose(
              LocalizedStrings.Connection_RUNNIOREADER_CAUGHT_SHUTDOWN.toLocalizedString());
        } catch (Exception ignore) {
        }
        return; // bug37520: exit loop (and thread)
      }
      logger.fatal(LocalizedMessage
          .create(LocalizedStrings.Connection_FAILED_SETTING_CHANNEL_TO_BLOCKING_MODE_0, ex));
      this.readerShuttingDown = true;
      try {
        requestClose(LocalizedStrings.Connection_FAILED_SETTING_CHANNEL_TO_BLOCKING_MODE_0
            .toLocalizedString(ex));
      } catch (Exception ignore) {
      }
      return;
    }

    if (!stopped) {
      // Assert.assertTrue(owner != null, "How did owner become null");
      if (logger.isDebugEnabled()) {
        logger.debug("Starting {}", p2pReaderName());
      }
    }
    // we should not change the state of the connection if we are a handshake reader thread
    // as there is a race between this thread and the application thread doing direct ack
    // fix for #40869
    boolean isHandShakeReader = false;
    try {
      for (;;) {
        if (stopped) {
          break;
        }
        if (SystemFailure.getFailure() != null) {
          // Allocate no objects here!
          Socket s = this.socket;
          if (s != null) {
            try {
              s.close();
            } catch (IOException e) {
              // don't care
            }
          }
          SystemFailure.checkFailure(); // throws
        }
        if (this.owner.getConduit().getCancelCriterion().isCancelInProgress()) {
          break;
        }

        try {
          ByteBuffer buff = getNIOBuffer();
          synchronized (stateLock) {
            connectionState = STATE_READING;
          }
          int amt = channel.read(buff);
          synchronized (stateLock) {
            connectionState = STATE_IDLE;
          }
          if (amt == 0) {
            continue;
          }
          if (amt < 0) {
            this.readerShuttingDown = true;
            try {
              requestClose("SocketChannel.read returned EOF");
              requestClose(
                  LocalizedStrings.Connection_SOCKETCHANNEL_READ_RETURNED_EOF.toLocalizedString());
            } catch (Exception e) {
              // ignore - shutting down
            }
            return;
          }

          processNIOBuffer();
          if (!this.isReceiver && (this.handshakeRead || this.handshakeCancelled)) {
            if (logger.isDebugEnabled()) {
              if (this.handshakeRead) {
                logger.debug("{} handshake has been read {}", p2pReaderName(), this);
              } else {
                logger.debug("{} handshake has been cancelled {}", p2pReaderName(), this);
              }
            }
            isHandShakeReader = true;
            // Once we have read the handshake the reader can go away
            break;
          }
        } catch (CancelException e) {
          if (logger.isDebugEnabled()) {
            logger.debug("{} Terminated <{}> due to cancellation", p2pReaderName(), this, e);
          }
          this.readerShuttingDown = true;
          try {
            requestClose(
                LocalizedStrings.Connection_CACHECLOSED_IN_CHANNEL_READ_0.toLocalizedString(e));
          } catch (Exception ex) {
          }
          return;
        } catch (ClosedChannelException e) {
          this.readerShuttingDown = true;
          try {
            requestClose(LocalizedStrings.Connection_CLOSEDCHANNELEXCEPTION_IN_CHANNEL_READ_0
                .toLocalizedString(e));
          } catch (Exception ex) {
          }
          return;
        } catch (IOException e) {
          if (!isSocketClosed() && !"Socket closed".equalsIgnoreCase(e.getMessage()) // needed for
                                                                                     // Solaris jdk
                                                                                     // 1.4.2_08
          ) {
            if (logger.isDebugEnabled() && !isIgnorableIOException(e)) {
              logger.debug("{} io exception for {}", p2pReaderName(), this, e);
            }
            if (e.getMessage().contains("interrupted by a call to WSACancelBlockingCall")) {
              if (logger.isDebugEnabled()) {
                logger.debug(
                    "{} received unexpected WSACancelBlockingCall exception, which may result in a hang",
                    p2pReaderName());
              }
            }
          }
          this.readerShuttingDown = true;
          try {
            requestClose(
                LocalizedStrings.Connection_IOEXCEPTION_IN_CHANNEL_READ_0.toLocalizedString(e));
          } catch (Exception ex) {
          }
          return;

        } catch (Exception e) {
          this.owner.getConduit().getCancelCriterion().checkCancelInProgress(null); // bug 37101
          if (!stopped && !isSocketClosed()) {
            logger.fatal(LocalizedMessage.create(
                LocalizedStrings.Connection_0_EXCEPTION_IN_CHANNEL_READ, p2pReaderName()), e);
          }
          this.readerShuttingDown = true;
          try {
            requestClose(
                LocalizedStrings.Connection_0_EXCEPTION_IN_CHANNEL_READ.toLocalizedString(e));
          } catch (Exception ex) {
          }
          return;
        }
      } // for
    } finally {
      if (!isHandShakeReader) {
        synchronized (stateLock) {
          connectionState = STATE_IDLE;
        }
      }
      if (logger.isDebugEnabled()) {
        logger.debug("{} runNioReader terminated id={} from {}", p2pReaderName(), conduitIdStr,
            remoteAddr);
      }
    }
  }

  /**
   * initiate suspect processing if a shared/ordered connection is lost and we're not shutting down
   */
  private void initiateSuspicionIfSharedUnordered() {
    if (this.isReceiver && this.handshakeRead && !this.preserveOrder && this.sharedResource) {
      if (!this.owner.getConduit().getCancelCriterion().isCancelInProgress()) {
        this.owner.getDM().getMembershipManager().suspectMember(this.getRemoteAddress(),
            INITIATING_SUSPECT_PROCESSING);
      }
    }
  }

  /**
   * checks to see if an exception should not be logged: i.e., "forcibly closed", "reset by peer",
   * or "connection reset"
   */
  public static boolean isIgnorableIOException(Exception e) {
    if (e instanceof ClosedChannelException) {
      return true;
    }

    String msg = e.getMessage();
    if (msg == null) {
      msg = e.toString();
    }

    msg = msg.toLowerCase();
    return (msg.contains("forcibly closed")) || (msg.contains("reset by peer"))
        || (msg.contains("connection reset"));
  }

  private static boolean validMsgType(int msgType) {
    return msgType == NORMAL_MSG_TYPE || msgType == CHUNKED_MSG_TYPE
        || msgType == END_CHUNKED_MSG_TYPE;
  }

  private void closeAllMsgDestreamers() {
    synchronized (this.destreamerLock) {
      if (this.idleMsgDestreamer != null) {
        this.idleMsgDestreamer.close();
        this.idleMsgDestreamer = null;
      }
      if (this.destreamerMap != null) {
        Iterator it = this.destreamerMap.values().iterator();
        while (it.hasNext()) {
          MsgDestreamer md = (MsgDestreamer) it.next();
          md.close();
        }
        this.destreamerMap = null;
      }
    }
  }

  MsgDestreamer obtainMsgDestreamer(short msgId, final Version v) {
    synchronized (this.destreamerLock) {
      if (this.destreamerMap == null) {
        this.destreamerMap = new HashMap();
      }
      Short key = new Short(msgId);
      MsgDestreamer result = (MsgDestreamer) this.destreamerMap.get(key);
      if (result == null) {
        result = this.idleMsgDestreamer;
        if (result != null) {
          this.idleMsgDestreamer = null;
        } else {
          result = new MsgDestreamer(this.owner.getConduit().getStats(),
              this.conduit.getCancelCriterion(), v);
        }
        result.setName(p2pReaderName() + " msgId=" + msgId);
        this.destreamerMap.put(key, result);
      }
      return result;
    }
  }

  void releaseMsgDestreamer(short msgId, MsgDestreamer md) {
    Short key = new Short(msgId);
    synchronized (this.destreamerLock) {
      this.destreamerMap.remove(key);
      if (this.idleMsgDestreamer == null) {
        md.reset();
        this.idleMsgDestreamer = md;
      } else {
        md.close();
      }
    }
  }

  private void sendFailureReply(int rpId, String exMsg, Throwable ex, boolean directAck) {
    ReplySender dm = null;
    if (directAck) {
      dm = new DirectReplySender(this);
    } else if (rpId != 0) {
      dm = this.owner.getDM();
    }
    if (dm != null) {
      ReplyMessage.send(getRemoteAddress(), rpId, new ReplyException(exMsg, ex), dm);
    }
  }

  private void runOioReader() {
    InputStream input = null;
    try {
      if (logger.isDebugEnabled()) {
        logger.debug("Socket is of type: {}", getSocket().getClass());
      }
      input = new BufferedInputStream(getSocket().getInputStream(), INITIAL_CAPACITY);
    } catch (IOException io) {
      if (stopped || owner.getConduit().getCancelCriterion().isCancelInProgress()) {
        return; // bug 37520: exit run loop (and thread)
      }
      logger.fatal(LocalizedMessage.create(LocalizedStrings.Connection_UNABLE_TO_GET_INPUT_STREAM),
          io);
      stopped = true;
    }

    if (!stopped) {
      Assert.assertTrue(owner != null,
          LocalizedStrings.Connection_OWNER_SHOULD_NOT_BE_NULL.toLocalizedString());
      if (logger.isDebugEnabled()) {
        logger.debug("Starting {}", p2pReaderName());
      }
    }

    byte[] headerBytes = new byte[MSG_HEADER_BYTES];

    final ByteArrayDataInput dis = new ByteArrayDataInput();
    while (!stopped) {
      try {
        if (SystemFailure.getFailure() != null) {
          // Allocate no objects here!
          Socket s = this.socket;
          if (s != null) {
            try {
              s.close();
            } catch (IOException e) {
              // don't care
            }
          }
          SystemFailure.checkFailure(); // throws
        }
        if (this.owner.getConduit().getCancelCriterion().isCancelInProgress()) {
          break;
        }
        int len = 0;
        if (readFully(input, headerBytes, headerBytes.length) < 0) {
          stopped = true;
          continue;
        }
        // long recvNanos = DistributionStats.getStatTime();
        len = ((headerBytes[MSG_HEADER_SIZE_OFFSET] & 0xff) * 0x1000000)
            + ((headerBytes[MSG_HEADER_SIZE_OFFSET + 1] & 0xff) * 0x10000)
            + ((headerBytes[MSG_HEADER_SIZE_OFFSET + 2] & 0xff) * 0x100)
            + (headerBytes[MSG_HEADER_SIZE_OFFSET + 3] & 0xff);
        /* byte msgHdrVersion = */ calcHdrVersion(len);
        len = calcMsgByteSize(len);
        int msgType = headerBytes[MSG_HEADER_TYPE_OFFSET];
        short msgId = (short) (((headerBytes[MSG_HEADER_ID_OFFSET] & 0xff) << 8)
            + (headerBytes[MSG_HEADER_ID_OFFSET + 1] & 0xff));
        boolean myDirectAck = (msgType & DIRECT_ACK_BIT) != 0;
        if (myDirectAck) {
          msgType &= ~DIRECT_ACK_BIT; // clear the bit
        }
        // Following validation fixes bug 31145
        if (!validMsgType(msgType)) {
          logger.fatal(LocalizedMessage.create(
              LocalizedStrings.Connection_UNKNOWN_P2P_MESSAGE_TYPE_0, Integer.valueOf(msgType)));
          this.readerShuttingDown = true;
          requestClose(LocalizedStrings.Connection_UNKNOWN_P2P_MESSAGE_TYPE_0
              .toLocalizedString(Integer.valueOf(msgType)));
          break;
        }
        if (logger.isTraceEnabled())
          logger.trace("{} reading {} bytes", conduitIdStr, len);
        byte[] bytes = new byte[len];
        if (readFully(input, bytes, len) < 0) {
          stopped = true;
          continue;
        }
        boolean interrupted = Thread.interrupted();
        try {
          if (this.handshakeRead) {
            if (msgType == NORMAL_MSG_TYPE) {
              // DMStats stats = this.owner.getConduit().stats;
              // long start = DistributionStats.getStatTime();
              this.owner.getConduit().getStats().incMessagesBeingReceived(true, len);
              dis.initialize(bytes, this.remoteVersion);
              DistributionMessage msg = null;
              try {
                ReplyProcessor21.initMessageRPId();
                long startSer = this.owner.getConduit().getStats().startMsgDeserialization();
                msg = (DistributionMessage) InternalDataSerializer.readDSFID(dis);
                this.owner.getConduit().getStats().endMsgDeserialization(startSer);
                if (dis.available() != 0) {
                  logger.warn(LocalizedMessage.create(
                      LocalizedStrings.Connection_MESSAGE_DESERIALIZATION_OF_0_DID_NOT_READ_1_BYTES,
                      new Object[] {msg, Integer.valueOf(dis.available())}));
                }
                // stats.incBatchCopyTime(start);
                try {
                  // start = DistributionStats.getStatTime();
                  if (!dispatchMessage(msg, len, myDirectAck)) {
                    continue;
                  }
                  // stats.incBatchSendTime(start);
                } catch (MemberShunnedException e) {
                  continue;
                } catch (Exception de) {
                  this.owner.getConduit().getCancelCriterion().checkCancelInProgress(de); // bug
                                                                                          // 37101
                  logger.fatal(LocalizedMessage
                      .create(LocalizedStrings.Connection_ERROR_DISPATCHING_MESSAGE), de);
                }
              } catch (VirtualMachineError err) {
                SystemFailure.initiateFailure(err);
                // If this ever returns, rethrow the error. We're poisoned
                // now, so don't let this thread continue.
                throw err;
              } catch (Throwable e) {
                // Whenever you catch Error or Throwable, you must also
                // catch VirtualMachineError (see above). However, there is
                // _still_ a possibility that you are dealing with a cascading
                // error condition, so you also need to check to see if the JVM
                // is still usable:
                SystemFailure.checkFailure();
                // In particular I want OutOfMem to be caught here
                if (!myDirectAck) {
                  String reason =
                      LocalizedStrings.Connection_ERROR_DESERIALIZING_MESSAGE.toLocalizedString();
                  sendFailureReply(ReplyProcessor21.getMessageRPId(), reason, e, myDirectAck);
                }
                if (e instanceof CancelException) {
                  if (!(e instanceof CacheClosedException)) {
                    // Just log a message if we had trouble deserializing due to
                    // CacheClosedException; see bug 43543
                    throw (CancelException) e;
                  }
                }
                logger.fatal(LocalizedMessage
                    .create(LocalizedStrings.Connection_ERROR_DESERIALIZING_MESSAGE), e);
                // requestClose();
                // return;
              } finally {
                ReplyProcessor21.clearMessageRPId();
              }
            } else if (msgType == CHUNKED_MSG_TYPE) {
              MsgDestreamer md = obtainMsgDestreamer(msgId, remoteVersion);
              this.owner.getConduit().getStats().incMessagesBeingReceived(md.size() == 0, len);
              try {
                md.addChunk(bytes);
              } catch (IOException ex) {
                logger.fatal(LocalizedMessage
                    .create(LocalizedStrings.Connection_FAILED_HANDLING_CHUNK_MESSAGE), ex);
              }
            } else /* (messageType == END_CHUNKED_MSG_TYPE) */ {
              MsgDestreamer md = obtainMsgDestreamer(msgId, remoteVersion);
              this.owner.getConduit().getStats().incMessagesBeingReceived(md.size() == 0, len);
              try {
                md.addChunk(bytes);
              } catch (IOException ex) {
                logger.fatal(LocalizedMessage
                    .create(LocalizedStrings.Connection_FAILED_HANDLING_END_CHUNK_MESSAGE), ex);
              }
              DistributionMessage msg = null;
              int msgLength = 0;
              String failureMsg = null;
              Throwable failureEx = null;
              int rpId = 0;
              try {
                msg = md.getMessage();
              } catch (ClassNotFoundException ex) {
                this.owner.getConduit().getStats().decMessagesBeingReceived(md.size());
                failureEx = ex;
                rpId = md.getRPid();
                logger.warn(LocalizedMessage
                    .create(LocalizedStrings.Connection_CLASSNOTFOUND_DESERIALIZING_MESSAGE_0, ex));
              } catch (IOException ex) {
                this.owner.getConduit().getStats().decMessagesBeingReceived(md.size());
                failureMsg = LocalizedStrings.Connection_IOEXCEPTION_DESERIALIZING_MESSAGE
                    .toLocalizedString();
                failureEx = ex;
                rpId = md.getRPid();
                logger.fatal(LocalizedMessage.create(
                    LocalizedStrings.Connection_IOEXCEPTION_DESERIALIZING_MESSAGE), failureEx);
              } catch (InterruptedException ex) {
                Thread.currentThread().interrupt();
                throw ex; // caught by outer try
              } catch (VirtualMachineError err) {
                SystemFailure.initiateFailure(err);
                // If this ever returns, rethrow the error. We're poisoned
                // now, so don't let this thread continue.
                throw err;
              } catch (Throwable ex) {
                // Whenever you catch Error or Throwable, you must also
                // catch VirtualMachineError (see above). However, there is
                // _still_ a possibility that you are dealing with a cascading
                // error condition, so you also need to check to see if the JVM
                // is still usable:
                SystemFailure.checkFailure();
                this.owner.getConduit().getStats().decMessagesBeingReceived(md.size());
                failureMsg = LocalizedStrings.Connection_UNEXPECTED_FAILURE_DESERIALIZING_MESSAGE
                    .toLocalizedString();
                failureEx = ex;
                rpId = md.getRPid();
                logger.fatal(
                    LocalizedMessage.create(
                        LocalizedStrings.Connection_UNEXPECTED_FAILURE_DESERIALIZING_MESSAGE),
                    failureEx);
              } finally {
                msgLength = md.size();
                releaseMsgDestreamer(msgId, md);
              }
              if (msg != null) {
                try {
                  if (!dispatchMessage(msg, msgLength, myDirectAck)) {
                    continue;
                  }
                } catch (MemberShunnedException e) {
                  continue;
                } catch (Exception de) {
                  this.owner.getConduit().getCancelCriterion().checkCancelInProgress(de);
                  logger.fatal(LocalizedMessage
                      .create(LocalizedStrings.Connection_ERROR_DISPATCHING_MESSAGE), de);
                } catch (ThreadDeath td) {
                  throw td;
                } catch (VirtualMachineError err) {
                  SystemFailure.initiateFailure(err);
                  // If this ever returns, rethrow the error. We're poisoned
                  // now, so don't let this thread continue.
                  throw err;
                } catch (Throwable t) {
                  // Whenever you catch Error or Throwable, you must also
                  // catch VirtualMachineError (see above). However, there is
                  // _still_ a possibility that you are dealing with a cascading
                  // error condition, so you also need to check to see if the JVM
                  // is still usable:
                  SystemFailure.checkFailure();
                  logger.fatal(LocalizedMessage
                      .create(LocalizedStrings.Connection_THROWABLE_DISPATCHING_MESSAGE), t);
                }
              } else if (failureEx != null) {
                sendFailureReply(rpId, failureMsg, failureEx, myDirectAck);
              }
            }
          } else {
            dis.initialize(bytes, null);
            if (!this.isReceiver) {
              this.replyCode = dis.readUnsignedByte();
              if (this.replyCode != REPLY_CODE_OK
                  && this.replyCode != REPLY_CODE_OK_WITH_ASYNC_INFO) {
                Integer replyCodeInteger = Integer.valueOf(this.replyCode);
                String err = LocalizedStrings.Connection_UNKNOWN_HANDSHAKE_REPLY_CODE_0
                    .toLocalizedString(replyCodeInteger);

                if (this.replyCode == 0) { // bug 37113
                  if (logger.isDebugEnabled()) {
                    logger.debug("{} (peer probably departed ungracefully)", err);
                  }
                } else {
                  logger.fatal(LocalizedMessage.create(
                      LocalizedStrings.Connection_UNKNOWN_HANDSHAKE_REPLY_CODE_0,
                      replyCodeInteger));
                }
                this.readerShuttingDown = true;
                requestClose(err);
                break;
              }
              if (this.replyCode == REPLY_CODE_OK_WITH_ASYNC_INFO) {
                this.asyncDistributionTimeout = dis.readInt();
                this.asyncQueueTimeout = dis.readInt();
                this.asyncMaxQueueSize = (long) dis.readInt() * (1024 * 1024);
                if (this.asyncDistributionTimeout != 0) {
                  logger.info(LocalizedMessage.create(
                      LocalizedStrings.Connection_0_ASYNC_CONFIGURATION_RECEIVED_1,
                      new Object[] {p2pReaderName(),
                          " asyncDistributionTimeout=" + this.asyncDistributionTimeout
                              + " asyncQueueTimeout=" + this.asyncQueueTimeout
                              + " asyncMaxQueueSize=" + (this.asyncMaxQueueSize / (1024 * 1024))}));
                }
                // read the product version ordinal for on-the-fly serialization
                // transformations (for rolling upgrades)
                this.remoteVersion = Version.readVersion(dis, true);
              }
              notifyHandshakeWaiter(true);
            } else {
              byte b = dis.readByte();
              if (b != 0) {
                throw new IllegalStateException(
                    LocalizedStrings.Connection_DETECTED_OLD_VERSION_PRE_5_0_1_OF_GEMFIRE_OR_NONGEMFIRE_DURING_HANDSHAKE_DUE_TO_INITIAL_BYTE_BEING_0
                        .toLocalizedString(new Byte(b)));
              }
              byte handshakeByte = dis.readByte();
              if (handshakeByte != HANDSHAKE_VERSION) {
                throw new IllegalStateException(
                    LocalizedStrings.Connection_DETECTED_WRONG_VERSION_OF_GEMFIRE_PRODUCT_DURING_HANDSHAKE_EXPECTED_0_BUT_FOUND_1
                        .toLocalizedString(
                            new Object[] {new Byte(HANDSHAKE_VERSION), new Byte(handshakeByte)}));
              }
              InternalDistributedMember remote = DSFIDFactory.readInternalDistributedMember(dis);
              setRemoteAddr(remote);
              Thread.currentThread().setName(LocalizedStrings.Connection_P2P_MESSAGE_READER_FOR_0
                  .toLocalizedString(this.remoteAddr, this.socket.getPort()));
              this.sharedResource = dis.readBoolean();
              this.preserveOrder = dis.readBoolean();
              this.uniqueId = dis.readLong();
              // read the product version ordinal for on-the-fly serialization
              // transformations (for rolling upgrades)
              this.remoteVersion = Version.readVersion(dis, true);
              int dominoNumber = 0;
              if (this.remoteVersion == null
                  || (this.remoteVersion.compareTo(Version.GFE_80) >= 0)) {
                dominoNumber = dis.readInt();
                if (this.sharedResource) {
                  dominoNumber = 0;
                }
                dominoCount.set(dominoNumber);
                // this.senderName = dis.readUTF();
                setThreadName(dominoNumber);
              }

              if (!this.sharedResource) {
                if (tipDomino()) {
                  logger.info(LocalizedMessage.create(
                      LocalizedStrings.Connection_THREAD_OWNED_RECEIVER_FORCING_ITSELF_TO_SEND_ON_THREAD_OWNED_SOCKETS));
                  // bug #49565 - if domino count is >= 2 use shared resources.
                  // Also see DistributedCacheOperation#supportsDirectAck
                } else { // if (dominoNumber < 2){
                  ConnectionTable.threadWantsOwnResources();
                  if (logger.isDebugEnabled()) {
                    logger.debug(
                        "thread-owned receiver with domino count of {} will prefer sending on thread-owned sockets",
                        dominoNumber);
                  }
                  // } else {
                  // ConnectionTable.threadWantsSharedResources();
                  // logger.fine("thread-owned receiver with domino count of " + dominoNumber + "
                  // will prefer shared sockets");
                }
                this.conduit.getStats().incThreadOwnedReceivers(1L, dominoNumber);
              }

              if (logger.isDebugEnabled()) {
                logger.debug("{} remoteAddr is {} {}", p2pReaderName(), this.remoteAddr,
                    (this.remoteVersion != null ? " (" + this.remoteVersion + ')' : ""));
              }

              String authInit = System.getProperty(
                  DistributionConfigImpl.SECURITY_SYSTEM_PREFIX + SECURITY_PEER_AUTH_INIT);
              boolean isSecure = authInit != null && authInit.length() != 0;

              if (isSecure) {
                // ARB: wait till member authentication has been confirmed?
                if (owner.getConduit().waitForMembershipCheck(this.remoteAddr)) {
                  sendOKHandshakeReply(); // fix for bug 33224
                  notifyHandshakeWaiter(true);
                } else {
                  // ARB: throw exception??
                  notifyHandshakeWaiter(false);
                  logger.warn(LocalizedMessage.create(
                      LocalizedStrings.Connection_0_TIMED_OUT_DURING_A_MEMBERSHIP_CHECK,
                      p2pReaderName()));
                }
              } else {
                sendOKHandshakeReply(); // fix for bug 33224
                notifyHandshakeWaiter(true);
              }
            }
            if (!this.isReceiver && (this.handshakeRead || this.handshakeCancelled)) {
              if (logger.isDebugEnabled()) {
                if (this.handshakeRead) {
                  logger.debug("{} handshake has been read {}", p2pReaderName(), this);
                } else {
                  logger.debug("{} handshake has been cancelled {}", p2pReaderName(), this);
                }
              }
              // Once we have read the handshake the reader can go away
              break;
            }
            continue;
          }
        } catch (InterruptedException e) {
          interrupted = true;
          this.owner.getConduit().getCancelCriterion().checkCancelInProgress(e);
          logger.fatal(LocalizedMessage.create(
              LocalizedStrings.Connection_0_STRAY_INTERRUPT_READING_MESSAGE, p2pReaderName()), e);
          continue;
        } catch (Exception ioe) {
          this.owner.getConduit().getCancelCriterion().checkCancelInProgress(ioe); // bug 37101
          if (!stopped) {
            logger.fatal(LocalizedMessage
                .create(LocalizedStrings.Connection_0_ERROR_READING_MESSAGE, p2pReaderName()), ioe);
          }
          continue;
        } finally {
          if (interrupted) {
            Thread.currentThread().interrupt();
          }
        }
      } catch (CancelException e) {
        if (logger.isDebugEnabled()) {
          String ccMsg = p2pReaderName() + " Cancelled: " + this;
          if (e.getMessage() != null) {
            ccMsg += ": " + e.getMessage();
          }
          logger.debug(ccMsg);
        }
        this.readerShuttingDown = true;
        try {
          requestClose(
              LocalizedStrings.Connection_CACHECLOSED_IN_CHANNEL_READ_0.toLocalizedString(e));
        } catch (Exception ex) {
        }
        this.stopped = true;
      } catch (IOException io) {
        boolean closed = isSocketClosed() || "Socket closed".equalsIgnoreCase(io.getMessage()); // needed
                                                                                                // for
                                                                                                // Solaris
                                                                                                // jdk
                                                                                                // 1.4.2_08
        if (!closed) {
          if (logger.isDebugEnabled() && !isIgnorableIOException(io)) {
            logger.debug("{} io exception for {}", p2pReaderName(), this, io);
          }
        }
        this.readerShuttingDown = true;
        try {
          requestClose(LocalizedStrings.Connection_IOEXCEPTION_RECEIVED_0.toLocalizedString(io));
        } catch (Exception ex) {
        }

        if (closed) {
          stopped = true;
        } else {
          // sleep a bit to avoid a hot error loop
          try {
            Thread.sleep(1000);
          } catch (InterruptedException ie) {
            Thread.currentThread().interrupt();
            if (this.owner.getConduit().getCancelCriterion().isCancelInProgress()) {
              return;
            }
            break;
          }
        }
      } // IOException
      catch (Exception e) {
        if (this.owner.getConduit().getCancelCriterion().isCancelInProgress()) {
          return; // bug 37101
        }
        if (!stopped && !(e instanceof InterruptedException)) {
          logger.fatal(LocalizedMessage.create(LocalizedStrings.Connection_0_EXCEPTION_RECEIVED,
              p2pReaderName()), e);
        }
        if (isSocketClosed()) {
          stopped = true;
        } else {
          this.readerShuttingDown = true;
          try {
            requestClose(LocalizedStrings.Connection_0_EXCEPTION_RECEIVED.toLocalizedString(e));
          } catch (Exception ex) {
          }

          // sleep a bit to avoid a hot error loop
          try {
            Thread.sleep(1000);
          } catch (InterruptedException ie) {
            Thread.currentThread().interrupt();
            break;
          }
        }
      }
    }
  }

  @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "DE_MIGHT_IGNORE")
  int readFully(InputStream input, byte[] buffer, int len) throws IOException {
    int bytesSoFar = 0;
    while (bytesSoFar < len) {
      this.owner.getConduit().getCancelCriterion().checkCancelInProgress(null);
      try {
        synchronized (stateLock) {
          connectionState = STATE_READING;
        }
        int bytesThisTime = input.read(buffer, bytesSoFar, len - bytesSoFar);
        if (bytesThisTime < 0) {
          this.readerShuttingDown = true;
          try {
            requestClose(LocalizedStrings.Connection_STREAM_READ_RETURNED_NONPOSITIVE_LENGTH
                .toLocalizedString());
          } catch (Exception ex) {
          }
          return -1;
        }
        bytesSoFar += bytesThisTime;
      } catch (InterruptedIOException io) {
        // Current thread has been interrupted. Regard it similar to an EOF
        this.readerShuttingDown = true;
        try {
          requestClose(LocalizedStrings.Connection_CURRENT_THREAD_INTERRUPTED.toLocalizedString());
        } catch (Exception ex) {
        }
        Thread.currentThread().interrupt();
        this.owner.getConduit().getCancelCriterion().checkCancelInProgress(null);
      } finally {
        synchronized (stateLock) {
          connectionState = STATE_IDLE;
        }
      }
    } // while
    return len;
  }

  /**
   * sends a serialized message to the other end of this connection. This is used by the
   * DirectChannel in GemFire when the message is going to be sent to multiple recipients.
   *
   * @throws ConnectionException if the conduit has stopped
   */
  public void sendPreserialized(ByteBuffer buffer, boolean cacheContentChanges,
      DistributionMessage msg) throws IOException, ConnectionException {
    if (!connected) {
      throw new ConnectionException(
          LocalizedStrings.Connection_NOT_CONNECTED_TO_0.toLocalizedString(this.remoteAddr));
    }
    if (this.batchFlusher != null) {
      batchSend(buffer);
      return;
    }
    final boolean origSocketInUse = this.socketInUse;
    byte originalState = -1;
    synchronized (stateLock) {
      originalState = this.connectionState;
      this.connectionState = STATE_SENDING;
    }
    this.socketInUse = true;
    try {
      if (useNIO()) {
        SocketChannel channel = getSocket().getChannel();
        nioWriteFully(channel, buffer, false, msg);
      } else {
        if (buffer.hasArray()) {
          this.output.write(buffer.array(), buffer.arrayOffset(),
              buffer.limit() - buffer.position());
        } else {
          byte[] bytesToWrite = getBytesToWrite(buffer);
          synchronized (outLock) {
            this.output.write(bytesToWrite);
            this.output.flush();
          }
        }
      }
      if (cacheContentChanges) {
        messagesSent++;
      }
    } finally {
      accessed();
      this.socketInUse = origSocketInUse;
      synchronized (stateLock) {
        this.connectionState = originalState;
      }
    }
  }

  /**
   * If <code>use</code> is true then "claim" the connection for our use. If <code>use</code> is
   * false then "release" the connection. Fixes bug 37657.
   *
   * @return true if connection was already in use at time of call; false if not.
   */
  public boolean setInUse(boolean use, long startTime, long ackWaitThreshold, long ackSAThreshold,
      List connectionGroup) {
    // just do the following; EVEN if the connection has been closed
    final boolean origSocketInUse = this.socketInUse;
    synchronized (this) {
      if (use && (ackWaitThreshold > 0 || ackSAThreshold > 0)) {
        // set times that events should be triggered
        this.transmissionStartTime = startTime;
        this.ackWaitTimeout = ackWaitThreshold;
        this.ackSATimeout = ackSAThreshold;
        this.ackConnectionGroup = connectionGroup;
        this.ackThreadName = Thread.currentThread().getName();
      } else {
        this.ackWaitTimeout = 0;
        this.ackSATimeout = 0;
        this.ackConnectionGroup = null;
        this.ackThreadName = null;
      }
      synchronized (this.stateLock) {
        this.connectionState = STATE_IDLE;
      }
      this.socketInUse = use;
    }
    if (!use) {
      accessed();
    }
    return origSocketInUse;
  }

  /**
   * For testing we want to configure the connection without having to read a handshake
   */
  protected void setSharedUnorderedForTest() {
    this.preserveOrder = false;
    this.sharedResource = true;
    this.handshakeRead = true;
  }


  /** ensure that a task is running to monitor transmission and reading of acks */
  public synchronized void scheduleAckTimeouts() {
    if (ackTimeoutTask == null) {
      final long msAW = this.owner.getDM().getConfig().getAckWaitThreshold() * 1000;
      final long msSA = this.owner.getDM().getConfig().getAckSevereAlertThreshold() * 1000;
      ackTimeoutTask = new SystemTimer.SystemTimerTask() {
        @Override
        public void run2() {
          if (owner.isClosed()) {
            return;
          }
          byte connState = -1;
          synchronized (stateLock) {
            connState = connectionState;
          }
          boolean sentAlert = false;
          synchronized (Connection.this) {
            if (socketInUse) {
              switch (connState) {
                case Connection.STATE_IDLE:
                  break;
                case Connection.STATE_SENDING:
                  sentAlert = doSevereAlertProcessing();
                  break;
                case Connection.STATE_POST_SENDING:
                  break;
                case Connection.STATE_READING_ACK:
                  sentAlert = doSevereAlertProcessing();
                  break;
                case Connection.STATE_RECEIVED_ACK:
                  break;
                default:
              }
            }
          }
          List group = ackConnectionGroup;
          if (sentAlert && group != null) {
            // since transmission and ack-receipt are performed serially, we don't
            // want to complain about all receivers out just because one was slow. We therefore
            // reset
            // the time stamps and give others more time
            for (Iterator it = group.iterator(); it.hasNext();) {
              Connection con = (Connection) it.next();
              if (con != Connection.this) {
                con.transmissionStartTime += con.ackSATimeout;
              }
            }
          }
        }
      };

      synchronized (owner) {
        SystemTimer timer = owner.getIdleConnTimer();
        if (timer != null) {
          if (msSA > 0) {
            timer.scheduleAtFixedRate(ackTimeoutTask, msAW, Math.min(msAW, msSA));
          } else {
            timer.schedule(ackTimeoutTask, msAW);
          }
        }
      }
    }
  }

  /** ack-wait-threshold and ack-severe-alert-threshold processing */
  protected boolean doSevereAlertProcessing() {
    long now = System.currentTimeMillis();
    if (ackSATimeout > 0 && (transmissionStartTime + ackWaitTimeout + ackSATimeout) <= now) {
      logger.fatal(LocalizedMessage.create(
          LocalizedStrings.Connection_0_SECONDS_HAVE_ELAPSED_WAITING_FOR_A_RESPONSE_FROM_1_FOR_THREAD_2,
          new Object[] {Long.valueOf((ackWaitTimeout + ackSATimeout) / 1000), getRemoteAddress(),
              ackThreadName}));
      // turn off subsequent checks by setting the timeout to zero, then boot the member
      ackSATimeout = 0;
      return true;
    } else if (!ackTimedOut && (0 < ackWaitTimeout)
        && (transmissionStartTime + ackWaitTimeout) <= now) {
      logger.warn(LocalizedMessage.create(
          LocalizedStrings.Connection_0_SECONDS_HAVE_ELAPSED_WAITING_FOR_A_RESPONSE_FROM_1_FOR_THREAD_2,
          new Object[] {Long.valueOf(ackWaitTimeout / 1000), getRemoteAddress(), ackThreadName}));
      ackTimedOut = true;

      final StringId state = (connectionState == Connection.STATE_SENDING)
          ? LocalizedStrings.Connection_TRANSMIT_ACKWAITTHRESHOLD
          : LocalizedStrings.Connection_RECEIVE_ACKWAITTHRESHOLD;
      if (ackSATimeout > 0) {
        this.owner.getDM().getMembershipManager()
            .suspectMembers(Collections.singleton(getRemoteAddress()), state.toLocalizedString());
      }
    }
    return false;
  }

  private static byte[] getBytesToWrite(ByteBuffer buffer) {
    byte[] bytesToWrite = new byte[buffer.limit()];
    buffer.get(bytesToWrite);
    return bytesToWrite;
  }

  private boolean addToQueue(ByteBuffer buffer, DistributionMessage msg, boolean force)
      throws ConnectionException {
    final DMStats stats = this.owner.getConduit().getStats();
    long start = DistributionStats.getStatTime();
    try {
      ConflationKey ck = null;
      if (msg != null) {
        ck = msg.getConflationKey();
      }
      Object objToQueue = null;
      // if we can conflate delay the copy to see if we can reuse
      // an already allocated buffer.
      final int newBytes = buffer.remaining();
      final int origBufferPos = buffer.position(); // to fix bug 34832
      if (ck == null || !ck.allowsConflation()) {
        // do this outside of sync for multi thread perf
        ByteBuffer newbb = ByteBuffer.allocate(newBytes);
        newbb.put(buffer);
        newbb.flip();
        objToQueue = newbb;
      }
      synchronized (this.outgoingQueue) {
        if (this.disconnectRequested) {
          buffer.position(origBufferPos);
          // we have given up so just drop this message.
          throw new ConnectionException(LocalizedStrings.Connection_FORCED_DISCONNECT_SENT_TO_0
              .toLocalizedString(this.remoteAddr));
        }
        if (!force && !this.asyncQueuingInProgress) {
          // reset buffer since we will be sending it. This fixes bug 34832
          buffer.position(origBufferPos);
          // the pusher emptied the queue so don't add since we are not forced to.
          return false;
        }
        boolean didConflation = false;
        if (ck != null) {
          if (ck.allowsConflation()) {
            objToQueue = ck;
            Object oldValue = this.conflatedKeys.put(ck, ck);
            if (oldValue != null) {
              ConflationKey oldck = (ConflationKey) oldValue;
              ByteBuffer oldBuffer = oldck.getBuffer();
              // need to always do this to allow old buffer to be gc'd
              oldck.setBuffer(null);
              // remove the conflated key from current spot in queue
              // Note we no longer remove from the queue because the search
              // can be expensive on large queues. Instead we just wait for
              // the queue removal code to find the oldck and ignore it since
              // its buffer is null
              // We do a quick check of the last thing in the queue
              // and if it has the same identity of our last thing then
              // remove it
              if (this.outgoingQueue.getLast() == oldck) {
                this.outgoingQueue.removeLast();
              }
              int oldBytes = oldBuffer.remaining();
              this.queuedBytes -= oldBytes;
              stats.incAsyncQueueSize(-oldBytes);
              stats.incAsyncConflatedMsgs();
              didConflation = true;
              if (oldBuffer.capacity() >= newBytes) {
                // copy new buffer into oldBuffer
                oldBuffer.clear();
                oldBuffer.put(buffer);
                oldBuffer.flip();
                ck.setBuffer(oldBuffer);
              } else {
                // old buffer was not large enough
                oldBuffer = null;
                ByteBuffer newbb = ByteBuffer.allocate(newBytes);
                newbb.put(buffer);
                newbb.flip();
                ck.setBuffer(newbb);
              }
            } else {
              // no old buffer so need to allocate one
              ByteBuffer newbb = ByteBuffer.allocate(newBytes);
              newbb.put(buffer);
              newbb.flip();
              ck.setBuffer(newbb);
            }
          } else {
            // just forget about having a conflatable operation
            /* Object removedVal = */ this.conflatedKeys.remove(ck);
          }
        }
        {
          long newQueueSize = newBytes + this.queuedBytes;
          if (newQueueSize > this.asyncMaxQueueSize) {
            logger.warn(LocalizedMessage.create(
                LocalizedStrings.Connection_QUEUED_BYTES_0_EXCEEDS_MAX_OF_1_ASKING_SLOW_RECEIVER_2_TO_DISCONNECT,
                new Object[] {newQueueSize, this.asyncMaxQueueSize, this.remoteAddr}));
            stats.incAsyncQueueSizeExceeded(1);
            disconnectSlowReceiver();
            // reset buffer since we will be sending it
            buffer.position(origBufferPos);
            return false;
          }
        }
        this.outgoingQueue.addLast(objToQueue);
        this.queuedBytes += newBytes;
        stats.incAsyncQueueSize(newBytes);
        if (!didConflation) {
          stats.incAsyncQueuedMsgs();
        }
        return true;
      }
    } finally {
      if (DistributionStats.enableClockStats) {
        stats.incAsyncQueueAddTime(DistributionStats.getStatTime() - start);
      }
    }
  }

  /**
   * Return true if it was able to handle a block write of the given buffer. Return false if it is
   * still the caller is still responsible for writing it.
   *
   * @throws ConnectionException if the conduit has stopped
   */
  private boolean handleBlockedWrite(ByteBuffer buffer, DistributionMessage msg)
      throws ConnectionException {
    if (!addToQueue(buffer, msg, true)) {
      return false;
    } else {
      startNioPusher();
      return true;
    }
  }

  private final Object nioPusherSync = new Object();

  private void startNioPusher() {
    synchronized (this.nioPusherSync) {
      while (this.pusherThread != null) {
        // wait for previous pusher thread to exit
        boolean interrupted = Thread.interrupted();
        try {
          this.nioPusherSync.wait(); // spurious wakeup ok
        } catch (InterruptedException ex) {
          interrupted = true;
          this.owner.getConduit().getCancelCriterion().checkCancelInProgress(ex);
        } finally {
          if (interrupted) {
            Thread.currentThread().interrupt();
          }
        }
      }
      this.asyncQueuingInProgress = true;
      ThreadGroup group = LoggingThreadGroup.createThreadGroup("P2P Writer Threads", logger);
      this.pusherThread = new Thread(group, new Runnable() {
        public void run() {
          Connection.this.runNioPusher();
        }
      }, "P2P async pusher to " + this.remoteAddr);
      this.pusherThread.setDaemon(true);
    } // synchronized
    this.pusherThread.start();
  }

  private ByteBuffer takeFromOutgoingQueue() throws InterruptedException {
    ByteBuffer result = null;
    final DMStats stats = this.owner.getConduit().getStats();
    long start = DistributionStats.getStatTime();
    try {
      synchronized (this.outgoingQueue) {
        if (this.disconnectRequested) {
          // don't bother with anymore work since we are done
          this.asyncQueuingInProgress = false;
          this.outgoingQueue.notifyAll();
          return null;
        }
        // Object o = this.outgoingQueue.poll();
        do {
          if (this.outgoingQueue.isEmpty()) {
            break;
          }
          Object o = this.outgoingQueue.removeFirst();
          if (o == null) {
            break;
          }
          if (o instanceof ConflationKey) {
            result = ((ConflationKey) o).getBuffer();
            if (result != null) {
              this.conflatedKeys.remove(o);
            } else {
              // if result is null then this same key will be found later in the
              // queue so we just need to skip this entry
              continue;
            }
          } else {
            result = (ByteBuffer) o;
          }
          int newBytes = result.remaining();
          this.queuedBytes -= newBytes;
          stats.incAsyncQueueSize(-newBytes);
          stats.incAsyncDequeuedMsgs();
        } while (result == null);
        if (result == null) {
          this.asyncQueuingInProgress = false;
          this.outgoingQueue.notifyAll();
        }
      }
      return result;
    } finally {
      if (DistributionStats.enableClockStats) {
        stats.incAsyncQueueRemoveTime(DistributionStats.getStatTime() - start);
      }
    }
  }

  private boolean disconnectRequested = false;


  /**
   * @since GemFire 4.2.2
   */
  private void disconnectSlowReceiver() {
    synchronized (this.outgoingQueue) {
      if (this.disconnectRequested) {
        // only ask once
        return;
      }
      this.disconnectRequested = true;
    }
    DistributionManager dm = this.owner.getDM();
    if (dm == null) {
      this.owner.removeEndpoint(this.remoteAddr,
          LocalizedStrings.Connection_NO_DISTRIBUTION_MANAGER.toLocalizedString());
      return;
    }
    dm.getMembershipManager().requestMemberRemoval(this.remoteAddr,
        LocalizedStrings.Connection_DISCONNECTED_AS_A_SLOWRECEIVER.toLocalizedString());
    // Ok, we sent the message, the coordinator should kick the member out
    // immediately and inform this process with a new view.
    // Let's wait
    // for that to happen and if it doesn't in X seconds
    // then remove the endpoint.
    final int FORCE_TIMEOUT = 3000;
    while (dm.getOtherDistributionManagerIds().contains(this.remoteAddr)) {
      try {
        Thread.sleep(50);
      } catch (InterruptedException ie) {
        Thread.currentThread().interrupt();
        this.owner.getConduit().getCancelCriterion().checkCancelInProgress(ie);
        return;
      }
    }
    this.owner.removeEndpoint(this.remoteAddr,
        LocalizedStrings.Connection_FORCE_DISCONNECT_TIMED_OUT.toLocalizedString());
    if (dm.getOtherDistributionManagerIds().contains(this.remoteAddr)) {
      if (logger.isDebugEnabled()) {
        logger.debug("Force disconnect timed out after waiting {} seconds", (FORCE_TIMEOUT / 1000));
      }
      return;
    }
  }

  /**
   * have the pusher thread check for queue overflow and for idle time exceeded
   */
  protected void runNioPusher() {
    try {
      final DMStats stats = this.owner.getConduit().getStats();
      final long threadStart = stats.startAsyncThread();
      try {
        stats.incAsyncQueues(1);
        stats.incAsyncThreads(1);

        try {
          int flushId = 0;
          while (this.asyncQueuingInProgress && this.connected) {
            if (SystemFailure.getFailure() != null) {
              // Allocate no objects here!
              Socket s = this.socket;
              if (s != null) {
                try {
                  s.close();
                } catch (IOException e) {
                  // don't care
                }
              }
              SystemFailure.checkFailure(); // throws
            }
            if (this.owner.getConduit().getCancelCriterion().isCancelInProgress()) {
              break;
            }
            flushId++;
            long flushStart = stats.startAsyncQueueFlush();
            try {
              long curQueuedBytes = this.queuedBytes;
              if (curQueuedBytes > this.asyncMaxQueueSize) {
                logger.warn(LocalizedMessage.create(
                    LocalizedStrings.Connection_QUEUED_BYTES_0_EXCEEDS_MAX_OF_1_ASKING_SLOW_RECEIVER_2_TO_DISCONNECT,
                    new Object[] {curQueuedBytes, this.asyncMaxQueueSize, this.remoteAddr}));
                stats.incAsyncQueueSizeExceeded(1);
                disconnectSlowReceiver();
                return;
              }
              SocketChannel channel = getSocket().getChannel();
              ByteBuffer bb = takeFromOutgoingQueue();
              if (bb == null) {
                if (logger.isDebugEnabled() && flushId == 1) {
                  logger.debug("P2P pusher found empty queue");
                }
                return;
              }
              nioWriteFully(channel, bb, true, null);
              // We should not add messagesSent here according to Bruce.
              // The counts are increased elsewhere.
              // messagesSent++;
              accessed();
            } finally {
              stats.endAsyncQueueFlush(flushStart);
            }
          } // while
        } finally {
          // need to force this to false before doing the requestClose calls
          synchronized (this.outgoingQueue) {
            this.asyncQueuingInProgress = false;
            this.outgoingQueue.notifyAll();
          }
        }
      } catch (InterruptedException ex) {
        // someone wants us to stop.
        // No need to set interrupt bit, we're quitting.
        // No need to throw an error, we're quitting.
      } catch (IOException ex) {
        final String err =
            LocalizedStrings.Connection_P2P_PUSHER_IO_EXCEPTION_FOR_0.toLocalizedString(this);
        if (!isSocketClosed()) {
          if (logger.isDebugEnabled() && !isIgnorableIOException(ex)) {
            logger.debug(err, ex);
          }
        }
        try {
          requestClose(err + ": " + ex);
        } catch (Exception ignore) {
        }
      } catch (CancelException ex) { // bug 37367
        final String err = LocalizedStrings.Connection_P2P_PUSHER_0_CAUGHT_CACHECLOSEDEXCEPTION_1
            .toLocalizedString(new Object[] {this, ex});
        logger.debug(err);
        try {
          requestClose(err);
        } catch (Exception ignore) {
        }
        return;
      } catch (Exception ex) {
        this.owner.getConduit().getCancelCriterion().checkCancelInProgress(ex); // bug 37101
        if (!isSocketClosed()) {
          logger.fatal(
              LocalizedMessage.create(LocalizedStrings.Connection_P2P_PUSHER_EXCEPTION_0, ex), ex);
        }
        try {
          requestClose(LocalizedStrings.Connection_P2P_PUSHER_EXCEPTION_0.toLocalizedString(ex));
        } catch (Exception ignore) {
        }
      } finally {
        stats.incAsyncQueueSize(-this.queuedBytes);
        this.queuedBytes = 0;
        stats.endAsyncThread(threadStart);
        stats.incAsyncThreads(-1);
        stats.incAsyncQueues(-1);
        if (logger.isDebugEnabled()) {
          logger.debug("runNioPusher terminated id={} from {}/{}", conduitIdStr, remoteAddr,
              remoteAddr);
        }
      }
    } finally {
      synchronized (this.nioPusherSync) {
        this.pusherThread = null;
        this.nioPusherSync.notify();
      }
    }
  }

  /**
   * Return false if socket writes to be done async/nonblocking Return true if socket writes to be
   * done sync/blocking
   */
  private boolean useSyncWrites(boolean forceAsync) {
    if (forceAsync) {
      return false;
    }
    // only use sync writes if:
    // we are already queuing
    if (this.asyncQueuingInProgress) {
      // it will just tack this msg onto the outgoing queue
      return true;
    }
    // or we are a receiver
    if (this.isReceiver) {
      return true;
    }
    // or we are an unordered connection
    if (!this.preserveOrder) {
      return true;
    }
    // or the receiver does not allow queuing
    if (this.asyncDistributionTimeout == 0) {
      return true;
    }
    // OTHERWISE return false and let caller send async
    return false;
  }

  /**
   * If true then act as if the socket buffer is full and start async queuing
   */
  public static volatile boolean FORCE_ASYNC_QUEUE = false;

  private static final int MAX_WAIT_TIME = (1 << 5); // ms (must be a power of 2)

  private void writeAsync(SocketChannel channel, ByteBuffer buffer, boolean forceAsync,
      DistributionMessage p_msg, final DMStats stats) throws IOException {
    DistributionMessage msg = p_msg;
    // async/non-blocking
    boolean socketWriteStarted = false;
    long startSocketWrite = 0;
    int retries = 0;
    int totalAmtWritten = 0;
    try {
      synchronized (this.outLock) {
        if (!forceAsync) {
          // check one more time while holding outLock in case a pusher was created
          if (this.asyncQueuingInProgress) {
            if (addToQueue(buffer, msg, false)) {
              return;
            }
            // fall through
          }
        }
        socketWriteStarted = true;
        startSocketWrite = stats.startSocketWrite(false);
        long now = System.currentTimeMillis();
        int waitTime = 1;
        long distributionTimeoutTarget = 0;
        // if asyncDistributionTimeout == 1 then we want to start queuing
        // as soon as we do a non blocking socket write that returns 0
        if (this.asyncDistributionTimeout != 1) {
          distributionTimeoutTarget = now + this.asyncDistributionTimeout;
        }
        long queueTimeoutTarget = now + this.asyncQueueTimeout;
        channel.configureBlocking(false);
        try {
          do {
            this.owner.getConduit().getCancelCriterion().checkCancelInProgress(null);
            retries++;
            int amtWritten;
            if (FORCE_ASYNC_QUEUE) {
              amtWritten = 0;
            } else {
              amtWritten = channel.write(buffer);
            }
            if (amtWritten == 0) {
              now = System.currentTimeMillis();
              long timeoutTarget;
              if (!forceAsync) {
                if (now > distributionTimeoutTarget) {
                  if (logger.isDebugEnabled()) {
                    if (distributionTimeoutTarget == 0) {
                      logger.debug(
                          "Starting async pusher to handle async queue because distribution-timeout is 1 and the last socket write would have blocked.");
                    } else {
                      long blockedMs = now - distributionTimeoutTarget;
                      blockedMs += this.asyncDistributionTimeout;
                      logger.debug(
                          "Blocked for {}ms which is longer than the max of {}ms so starting async pusher to handle async queue.",
                          blockedMs, this.asyncDistributionTimeout);
                    }
                  }
                  stats.incAsyncDistributionTimeoutExceeded();
                  if (totalAmtWritten > 0) {
                    // we have written part of the msg to the socket buffer
                    // and we are going to queue the remainder.
                    // We set msg to null so that will not make
                    // the partial msg a candidate for conflation.
                    msg = null;
                  }
                  if (handleBlockedWrite(buffer, msg)) {
                    return;
                  }
                }
                timeoutTarget = distributionTimeoutTarget;
              } else {
                boolean disconnectNeeded = false;
                long curQueuedBytes = this.queuedBytes;
                if (curQueuedBytes > this.asyncMaxQueueSize) {
                  logger.warn(LocalizedMessage.create(
                      LocalizedStrings.Connection_QUEUED_BYTES_0_EXCEEDS_MAX_OF_1_ASKING_SLOW_RECEIVER_2_TO_DISCONNECT,
                      new Object[] {Long.valueOf(curQueuedBytes),
                          Long.valueOf(this.asyncMaxQueueSize), this.remoteAddr}));
                  stats.incAsyncQueueSizeExceeded(1);
                  disconnectNeeded = true;
                }
                if (now > queueTimeoutTarget) {
                  // we have waited long enough
                  // the pusher has been idle too long!
                  long blockedMs = now - queueTimeoutTarget;
                  blockedMs += this.asyncQueueTimeout;
                  logger.warn(LocalizedMessage.create(
                      LocalizedStrings.Connection_BLOCKED_FOR_0_MS_WHICH_IS_LONGER_THAN_THE_MAX_OF_1_MS_ASKING_SLOW_RECEIVER_2_TO_DISCONNECT,
                      new Object[] {Long.valueOf(blockedMs),
                          Integer.valueOf(this.asyncQueueTimeout), this.remoteAddr}));
                  stats.incAsyncQueueTimeouts(1);
                  disconnectNeeded = true;
                }
                if (disconnectNeeded) {
                  disconnectSlowReceiver();
                  synchronized (this.outgoingQueue) {
                    this.asyncQueuingInProgress = false;
                    this.outgoingQueue.notifyAll(); // for bug 42330
                  }
                  return;
                }
                timeoutTarget = queueTimeoutTarget;
              }
              {
                long msToWait = waitTime;
                long msRemaining = timeoutTarget - now;
                if (msRemaining > 0) {
                  msRemaining /= 2;
                }
                if (msRemaining < msToWait) {
                  msToWait = msRemaining;
                }
                if (msToWait <= 0) {
                  Thread.yield();
                } else {
                  boolean interrupted = Thread.interrupted();
                  try {
                    Thread.sleep(msToWait);
                  } catch (InterruptedException ex) {
                    interrupted = true;
                    this.owner.getConduit().getCancelCriterion().checkCancelInProgress(ex);
                  } finally {
                    if (interrupted) {
                      Thread.currentThread().interrupt();
                    }
                  }
                }
              }
              if (waitTime < MAX_WAIT_TIME) {
                // double it since it is not yet the max
                waitTime <<= 1;
              }
            } // amtWritten == 0
            else {
              totalAmtWritten += amtWritten;
              // reset queueTimeoutTarget since we made some progress
              queueTimeoutTarget = System.currentTimeMillis() + this.asyncQueueTimeout;
              waitTime = 1;
            }
          } while (buffer.remaining() > 0);
        } finally {
          channel.configureBlocking(true);
        }
      }
    } finally {
      if (socketWriteStarted) {
        if (retries > 0) {
          retries--;
        }
        stats.endSocketWrite(false, startSocketWrite, totalAmtWritten, retries);
      }
    }
  }

  /**
   * nioWriteFully implements a blocking write on a channel that is in non-blocking mode.
   *
   * @param forceAsync true if we need to force a blocking async write.
   * @throws ConnectionException if the conduit has stopped
   */
  protected void nioWriteFully(SocketChannel channel, ByteBuffer buffer, boolean forceAsync,
      DistributionMessage msg) throws IOException, ConnectionException {
    final DMStats stats = this.owner.getConduit().getStats();
    if (!this.sharedResource) {
      stats.incTOSentMsg();
    }
    if (useSyncWrites(forceAsync)) {
      if (this.asyncQueuingInProgress) {
        if (addToQueue(buffer, msg, false)) {
          return;
        }
        // fall through
      }
      long startLock = stats.startSocketLock();
      synchronized (this.outLock) {
        stats.endSocketLock(startLock);
        if (this.asyncQueuingInProgress) {
          if (addToQueue(buffer, msg, false)) {
            return;
          }
          // fall through
        }
        do {
          int amtWritten = 0;
          long start = stats.startSocketWrite(true);
          try {
            // this.writerThread = Thread.currentThread();
            amtWritten = channel.write(buffer);
          } finally {
            stats.endSocketWrite(true, start, amtWritten, 0);
            // this.writerThread = null;
          }
        } while (buffer.remaining() > 0);
      } // synchronized
    } else {
      writeAsync(channel, buffer, forceAsync, msg, stats);
    }
  }

  /** gets the buffer for receiving message length bytes */
  protected ByteBuffer getNIOBuffer() {
    final DMStats stats = this.owner.getConduit().getStats();
    if (nioInputBuffer == null) {
      int allocSize = this.recvBufferSize;
      if (allocSize == -1) {
        allocSize = this.owner.getConduit().tcpBufferSize;
      }
      nioInputBuffer = Buffers.acquireReceiveBuffer(allocSize, stats);
    }
    return nioInputBuffer;
  }

  /**
   * stateLock is used to synchronize state changes.
   */
  private final Object stateLock = new Object();

  /** for timeout processing, this is the current state of the connection */
  private byte connectionState = STATE_IDLE;

  /* ~~~~~~~~~~~~~ connection states ~~~~~~~~~~~~~~~ */
  /** the connection is idle, but may be in use */
  protected static final byte STATE_IDLE = 0;
  /** the connection is in use and is transmitting data */
  protected static final byte STATE_SENDING = 1;
  /** the connection is in use and is done transmitting */
  protected static final byte STATE_POST_SENDING = 2;
  /** the connection is in use and is reading a direct-ack */
  protected static final byte STATE_READING_ACK = 3;
  /** the connection is in use and has finished reading a direct-ack */
  protected static final byte STATE_RECEIVED_ACK = 4;
  /** the connection is in use and is reading a message */
  protected static final byte STATE_READING = 5;
  /* ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */

  /** set to true if we exceeded the ack-wait-threshold waiting for a response */
  protected volatile boolean ackTimedOut;

  /**
   * @param msToWait number of milliseconds to wait for an ack. If 0 then wait forever.
   * @param msInterval interval between checks
   * @throws SocketTimeoutException if msToWait expires.
   * @throws ConnectionException if ack is not received (fixes bug 34312)
   */
  public void readAck(final int msToWait, final long msInterval,
      final DirectReplyProcessor processor) throws SocketTimeoutException, ConnectionException {
    if (isSocketClosed()) {
      throw new ConnectionException(
          LocalizedStrings.Connection_CONNECTION_IS_CLOSED.toLocalizedString());
    }
    synchronized (this.stateLock) {
      this.connectionState = STATE_READING_ACK;
    }

    boolean origSocketInUse = this.socketInUse;
    this.socketInUse = true;
    MsgReader msgReader = null;
    DMStats stats = owner.getConduit().getStats();
    final Version version = getRemoteVersion();
    try {
      if (useNIO()) {
        msgReader = new NIOMsgReader(this, version);
      } else {
        msgReader = new OioMsgReader(this, version);
      }

      Header header = msgReader.readHeader();

      ReplyMessage msg;
      int len;
      if (header.getNioMessageType() == NORMAL_MSG_TYPE) {
        msg = (ReplyMessage) msgReader.readMessage(header);
        len = header.getNioMessageLength();
      } else {
        MsgDestreamer destreamer = obtainMsgDestreamer(header.getNioMessageId(), version);
        while (header.getNioMessageType() == CHUNKED_MSG_TYPE) {
          msgReader.readChunk(header, destreamer);
          header = msgReader.readHeader();
        }
        msgReader.readChunk(header, destreamer);
        msg = (ReplyMessage) destreamer.getMessage();
        releaseMsgDestreamer(header.getNioMessageId(), destreamer);
        len = destreamer.size();
      }
      // I'd really just like to call dispatchMessage here. However,
      // that call goes through a bunch of checks that knock about
      // 10% of the performance. Since this direct-ack stuff is all
      // about performance, we'll skip those checks. Skipping them
      // should be legit, because we just sent a message so we know
      // the member is already in our view, etc.
      ClusterDistributionManager dm = (ClusterDistributionManager) owner.getDM();
      msg.setBytesRead(len);
      msg.setSender(remoteAddr);
      stats.incReceivedMessages(1L);
      stats.incReceivedBytes(msg.getBytesRead());
      stats.incMessageChannelTime(msg.resetTimestamp());
      msg.process(dm, processor);
      // dispatchMessage(msg, len, false);
    } catch (MemberShunnedException e) {
      // do nothing
    } catch (SocketTimeoutException timeout) {
      throw timeout;
    } catch (IOException e) {
      final String err =
          LocalizedStrings.Connection_ACK_READ_IO_EXCEPTION_FOR_0.toLocalizedString(this);
      if (!isSocketClosed()) {
        if (logger.isDebugEnabled() && !isIgnorableIOException(e)) {
          logger.debug(err, e);
        }
      }
      try {
        requestClose(err + ": " + e);
      } catch (Exception ex) {
      }
      throw new ConnectionException(
          LocalizedStrings.Connection_UNABLE_TO_READ_DIRECT_ACK_BECAUSE_0.toLocalizedString(e));
    } catch (ConnectionException e) {
      this.owner.getConduit().getCancelCriterion().checkCancelInProgress(e);
      throw e;
    } catch (Exception e) {
      this.owner.getConduit().getCancelCriterion().checkCancelInProgress(e);
      if (!isSocketClosed()) {
        logger.fatal(LocalizedMessage.create(LocalizedStrings.Connection_ACK_READ_EXCEPTION), e);
      }
      try {
        requestClose(LocalizedStrings.Connection_ACK_READ_EXCEPTION_0.toLocalizedString(e));
      } catch (Exception ex) {
      }
      throw new ConnectionException(
          LocalizedStrings.Connection_UNABLE_TO_READ_DIRECT_ACK_BECAUSE_0.toLocalizedString(e));
    } finally {
      stats.incProcessedMessages(1L);
      accessed();
      this.socketInUse = origSocketInUse;
      if (this.ackTimedOut) {
        logger.info(
            LocalizedMessage.create(LocalizedStrings.Connection_FINISHED_WAITING_FOR_REPLY_FROM_0,
                new Object[] {getRemoteAddress()}));
        this.ackTimedOut = false;
      }
      if (msgReader != null) {
        msgReader.close();
      }
    }
    synchronized (stateLock) {
      this.connectionState = STATE_RECEIVED_ACK;
    }
  }

  /**
   * processes the current NIO buffer. If there are complete messages in the buffer, they are
   * deserialized and passed to TCPConduit for further processing
   */
  private void processNIOBuffer() throws ConnectionException, IOException {
    if (nioInputBuffer != null) {
      nioInputBuffer.flip();
    }
    boolean done = false;

    while (!done && connected) {
      this.owner.getConduit().getCancelCriterion().checkCancelInProgress(null);
      // long startTime = DistributionStats.getStatTime();
      int remaining = nioInputBuffer.remaining();
      if (nioLengthSet || remaining >= MSG_HEADER_BYTES) {
        if (!nioLengthSet) {
          int headerStartPos = nioInputBuffer.position();
          nioMessageLength = nioInputBuffer.getInt();
          /* nioMessageVersion = */ calcHdrVersion(nioMessageLength);
          nioMessageLength = calcMsgByteSize(nioMessageLength);
          nioMessageType = nioInputBuffer.get();
          nioMsgId = nioInputBuffer.getShort();
          directAck = (nioMessageType & DIRECT_ACK_BIT) != 0;
          if (directAck) {
            nioMessageType &= ~DIRECT_ACK_BIT; // clear the ack bit
          }
          // Following validation fixes bug 31145
          if (!validMsgType(nioMessageType)) {
            Integer nioMessageTypeInteger = Integer.valueOf(nioMessageType);
            logger.fatal(LocalizedMessage.create(
                LocalizedStrings.Connection_UNKNOWN_P2P_MESSAGE_TYPE_0, nioMessageTypeInteger));
            this.readerShuttingDown = true;
            requestClose(LocalizedStrings.Connection_UNKNOWN_P2P_MESSAGE_TYPE_0
                .toLocalizedString(nioMessageTypeInteger));
            break;
          }
          nioLengthSet = true;
          // keep the header "in" the buffer until we have read the entire msg.
          // Trust me: this will reduce copying on large messages.
          nioInputBuffer.position(headerStartPos);
        }
        if (remaining >= nioMessageLength + MSG_HEADER_BYTES) {
          nioLengthSet = false;
          nioInputBuffer.position(nioInputBuffer.position() + MSG_HEADER_BYTES);
          // don't trust the message deserialization to leave the position in
          // the correct spot. Some of the serialization uses buffered
          // streams that can leave the position at the wrong spot
          int startPos = nioInputBuffer.position();
          int oldLimit = nioInputBuffer.limit();
          nioInputBuffer.limit(startPos + nioMessageLength);
          if (this.handshakeRead) {
            if (nioMessageType == NORMAL_MSG_TYPE) {
              this.owner.getConduit().getStats().incMessagesBeingReceived(true, nioMessageLength);
              ByteBufferInputStream bbis =
                  remoteVersion == null ? new ByteBufferInputStream(nioInputBuffer)
                      : new VersionedByteBufferInputStream(nioInputBuffer, remoteVersion);
              DistributionMessage msg = null;
              try {
                ReplyProcessor21.initMessageRPId();
                // add serialization stats
                long startSer = this.owner.getConduit().getStats().startMsgDeserialization();
                msg = (DistributionMessage) InternalDataSerializer.readDSFID(bbis);
                this.owner.getConduit().getStats().endMsgDeserialization(startSer);
                if (bbis.available() != 0) {
                  logger.warn(LocalizedMessage.create(
                      LocalizedStrings.Connection_MESSAGE_DESERIALIZATION_OF_0_DID_NOT_READ_1_BYTES,
                      new Object[] {msg, Integer.valueOf(bbis.available())}));
                }
                try {
                  if (!dispatchMessage(msg, nioMessageLength, directAck)) {
                    directAck = false;
                  }
                } catch (MemberShunnedException e) {
                  directAck = false; // don't respond (bug39117)
                } catch (Exception de) {
                  this.owner.getConduit().getCancelCriterion().checkCancelInProgress(de);
                  logger.fatal(LocalizedMessage
                      .create(LocalizedStrings.Connection_ERROR_DISPATCHING_MESSAGE), de);
                } catch (ThreadDeath td) {
                  throw td;
                } catch (VirtualMachineError err) {
                  SystemFailure.initiateFailure(err);
                  // If this ever returns, rethrow the error. We're poisoned
                  // now, so don't let this thread continue.
                  throw err;
                } catch (Throwable t) {
                  // Whenever you catch Error or Throwable, you must also
                  // catch VirtualMachineError (see above). However, there is
                  // _still_ a possibility that you are dealing with a cascading
                  // error condition, so you also need to check to see if the JVM
                  // is still usable:
                  SystemFailure.checkFailure();
                  logger.fatal(LocalizedMessage
                      .create(LocalizedStrings.Connection_THROWABLE_DISPATCHING_MESSAGE), t);
                }
              } catch (VirtualMachineError err) {
                SystemFailure.initiateFailure(err);
                // If this ever returns, rethrow the error. We're poisoned
                // now, so don't let this thread continue.
                throw err;
              } catch (Throwable t) {
                // Whenever you catch Error or Throwable, you must also
                // catch VirtualMachineError (see above). However, there is
                // _still_ a possibility that you are dealing with a cascading
                // error condition, so you also need to check to see if the JVM
                // is still usable:
                SystemFailure.checkFailure();
                sendFailureReply(ReplyProcessor21.getMessageRPId(),
                    LocalizedStrings.Connection_ERROR_DESERIALIZING_MESSAGE.toLocalizedString(), t,
                    directAck);
                if (t instanceof ThreadDeath) {
                  throw (ThreadDeath) t;
                }
                if (t instanceof CancelException) {
                  if (!(t instanceof CacheClosedException)) {
                    // Just log a message if we had trouble deserializing due to
                    // CacheClosedException; see bug 43543
                    throw (CancelException) t;
                  }
                }
                logger.fatal(LocalizedMessage
                    .create(LocalizedStrings.Connection_ERROR_DESERIALIZING_MESSAGE), t);
              } finally {
                ReplyProcessor21.clearMessageRPId();
              }
            } else if (nioMessageType == CHUNKED_MSG_TYPE) {
              MsgDestreamer md = obtainMsgDestreamer(nioMsgId, remoteVersion);
              this.owner.getConduit().getStats().incMessagesBeingReceived(md.size() == 0,
                  nioMessageLength);
              try {
                md.addChunk(nioInputBuffer, nioMessageLength);
              } catch (IOException ex) {
                logger.fatal(LocalizedMessage
                    .create(LocalizedStrings.Connection_FAILED_HANDLING_CHUNK_MESSAGE), ex);
              }
            } else /* (nioMessageType == END_CHUNKED_MSG_TYPE) */ {
              // logger.info("END_CHUNK msgId="+nioMsgId);
              MsgDestreamer md = obtainMsgDestreamer(nioMsgId, remoteVersion);
              this.owner.getConduit().getStats().incMessagesBeingReceived(md.size() == 0,
                  nioMessageLength);
              try {
                md.addChunk(nioInputBuffer, nioMessageLength);
              } catch (IOException ex) {
                logger.fatal(LocalizedMessage
                    .create(LocalizedStrings.Connection_FAILED_HANDLING_END_CHUNK_MESSAGE), ex);
              }
              DistributionMessage msg = null;
              int msgLength = 0;
              String failureMsg = null;
              Throwable failureEx = null;
              int rpId = 0;
              boolean interrupted = false;
              try {
                msg = md.getMessage();
              } catch (ClassNotFoundException ex) {
                this.owner.getConduit().getStats().decMessagesBeingReceived(md.size());
                failureMsg = LocalizedStrings.Connection_CLASSNOTFOUND_DESERIALIZING_MESSAGE
                    .toLocalizedString();
                failureEx = ex;
                rpId = md.getRPid();
                logger.fatal(LocalizedMessage
                    .create(LocalizedStrings.Connection_CLASSNOTFOUND_DESERIALIZING_MESSAGE_0, ex));
              } catch (IOException ex) {
                this.owner.getConduit().getStats().decMessagesBeingReceived(md.size());
                failureMsg = LocalizedStrings.Connection_IOEXCEPTION_DESERIALIZING_MESSAGE
                    .toLocalizedString();
                failureEx = ex;
                rpId = md.getRPid();
                logger.fatal(LocalizedMessage.create(
                    LocalizedStrings.Connection_IOEXCEPTION_DESERIALIZING_MESSAGE), failureEx);
              } catch (InterruptedException ex) {
                interrupted = true;
                this.owner.getConduit().getCancelCriterion().checkCancelInProgress(ex);
              } catch (VirtualMachineError err) {
                SystemFailure.initiateFailure(err);
                // If this ever returns, rethrow the error. We're poisoned
                // now, so don't let this thread continue.
                throw err;
              } catch (Throwable ex) {
                // Whenever you catch Error or Throwable, you must also
                // catch VirtualMachineError (see above). However, there is
                // _still_ a possibility that you are dealing with a cascading
                // error condition, so you also need to check to see if the JVM
                // is still usable:
                SystemFailure.checkFailure();
                this.owner.getConduit().getCancelCriterion().checkCancelInProgress(ex);
                this.owner.getConduit().getStats().decMessagesBeingReceived(md.size());
                failureMsg = LocalizedStrings.Connection_UNEXPECTED_FAILURE_DESERIALIZING_MESSAGE
                    .toLocalizedString();
                failureEx = ex;
                rpId = md.getRPid();
                logger.fatal(
                    LocalizedMessage.create(
                        LocalizedStrings.Connection_UNEXPECTED_FAILURE_DESERIALIZING_MESSAGE),
                    failureEx);
              } finally {
                msgLength = md.size();
                releaseMsgDestreamer(nioMsgId, md);
                if (interrupted) {
                  Thread.currentThread().interrupt();
                }
              }
              if (msg != null) {
                try {
                  if (!dispatchMessage(msg, msgLength, directAck)) {
                    directAck = false;
                  }
                } catch (MemberShunnedException e) {
                  // not a member anymore - don't reply
                  directAck = false;
                } catch (Exception de) {
                  this.owner.getConduit().getCancelCriterion().checkCancelInProgress(de);
                  logger.fatal(LocalizedMessage
                      .create(LocalizedStrings.Connection_ERROR_DISPATCHING_MESSAGE), de);
                } catch (ThreadDeath td) {
                  throw td;
                } catch (VirtualMachineError err) {
                  SystemFailure.initiateFailure(err);
                  // If this ever returns, rethrow the error. We're poisoned
                  // now, so don't let this thread continue.
                  throw err;
                } catch (Throwable t) {
                  // Whenever you catch Error or Throwable, you must also
                  // catch VirtualMachineError (see above). However, there is
                  // _still_ a possibility that you are dealing with a cascading
                  // error condition, so you also need to check to see if the JVM
                  // is still usable:
                  SystemFailure.checkFailure();
                  logger.fatal(LocalizedMessage
                      .create(LocalizedStrings.Connection_THROWABLE_DISPATCHING_MESSAGE), t);
                }
              } else if (failureEx != null) {
                sendFailureReply(rpId, failureMsg, failureEx, directAck);
              }
            }
          } else {
            // read HANDSHAKE
            ByteBufferInputStream bbis = new ByteBufferInputStream(nioInputBuffer);
            DataInputStream dis = new DataInputStream(bbis);
            if (!this.isReceiver) {
              try {
                this.replyCode = dis.readUnsignedByte();
                if (this.replyCode == REPLY_CODE_OK_WITH_ASYNC_INFO) {
                  this.asyncDistributionTimeout = dis.readInt();
                  this.asyncQueueTimeout = dis.readInt();
                  this.asyncMaxQueueSize = (long) dis.readInt() * (1024 * 1024);
                  if (this.asyncDistributionTimeout != 0) {
                    logger.info(LocalizedMessage.create(
                        LocalizedStrings.Connection_0_ASYNC_CONFIGURATION_RECEIVED_1,
                        new Object[] {p2pReaderName(),
                            " asyncDistributionTimeout=" + this.asyncDistributionTimeout
                                + " asyncQueueTimeout=" + this.asyncQueueTimeout
                                + " asyncMaxQueueSize="
                                + (this.asyncMaxQueueSize / (1024 * 1024))}));
                  }
                  // read the product version ordinal for on-the-fly serialization
                  // transformations (for rolling upgrades)
                  this.remoteVersion = Version.readVersion(dis, true);
                }
              } catch (Exception e) {
                this.owner.getConduit().getCancelCriterion().checkCancelInProgress(e);
                logger.fatal(LocalizedMessage.create(
                    LocalizedStrings.Connection_ERROR_DESERIALIZING_P2P_HANDSHAKE_REPLY), e);
                this.readerShuttingDown = true;
                requestClose(LocalizedStrings.Connection_ERROR_DESERIALIZING_P2P_HANDSHAKE_REPLY
                    .toLocalizedString());
                return;
              } catch (ThreadDeath td) {
                throw td;
              } catch (VirtualMachineError err) {
                SystemFailure.initiateFailure(err);
                // If this ever returns, rethrow the error. We're poisoned
                // now, so don't let this thread continue.
                throw err;
              } catch (Throwable t) {
                // Whenever you catch Error or Throwable, you must also
                // catch VirtualMachineError (see above). However, there is
                // _still_ a possibility that you are dealing with a cascading
                // error condition, so you also need to check to see if the JVM
                // is still usable:
                SystemFailure.checkFailure();
                logger.fatal(
                    LocalizedMessage.create(
                        LocalizedStrings.Connection_THROWABLE_DESERIALIZING_P2P_HANDSHAKE_REPLY),
                    t);
                this.readerShuttingDown = true;
                requestClose(LocalizedStrings.Connection_THROWABLE_DESERIALIZING_P2P_HANDSHAKE_REPLY
                    .toLocalizedString());
                return;
              }
              if (this.replyCode != REPLY_CODE_OK
                  && this.replyCode != REPLY_CODE_OK_WITH_ASYNC_INFO) {
                StringId err =
                    LocalizedStrings.Connection_UNKNOWN_HANDSHAKE_REPLY_CODE_0_NIOMESSAGELENGTH_1_PROCESSORTYPE_2;
                Object[] errArgs = new Object[] {Integer.valueOf(this.replyCode),
                    Integer.valueOf(nioMessageLength)};
                if (replyCode == 0 && logger.isDebugEnabled()) { // bug 37113
                  logger.debug(
                      err.toLocalizedString(errArgs) + " (peer probably departed ungracefully)");
                } else {
                  logger.fatal(LocalizedMessage.create(err, errArgs));
                }
                this.readerShuttingDown = true;
                requestClose(err.toLocalizedString(errArgs));
                return;
              }
              notifyHandshakeWaiter(true);
            } else {
              try {
                byte b = dis.readByte();
                if (b != 0) {
                  throw new IllegalStateException(
                      LocalizedStrings.Connection_DETECTED_OLD_VERSION_PRE_501_OF_GEMFIRE_OR_NONGEMFIRE_DURING_HANDSHAKE_DUE_TO_INITIAL_BYTE_BEING_0
                          .toLocalizedString(new Byte(b)));
                }
                byte handshakeByte = dis.readByte();
                if (handshakeByte != HANDSHAKE_VERSION) {
                  throw new IllegalStateException(
                      LocalizedStrings.Connection_DETECTED_WRONG_VERSION_OF_GEMFIRE_PRODUCT_DURING_HANDSHAKE_EXPECTED_0_BUT_FOUND_1
                          .toLocalizedString(
                              new Object[] {new Byte(HANDSHAKE_VERSION), new Byte(handshakeByte)}));
                }
                InternalDistributedMember remote = DSFIDFactory.readInternalDistributedMember(dis);
                setRemoteAddr(remote);
                this.sharedResource = dis.readBoolean();
                this.preserveOrder = dis.readBoolean();
                this.uniqueId = dis.readLong();
                // read the product version ordinal for on-the-fly serialization
                // transformations (for rolling upgrades)
                this.remoteVersion = Version.readVersion(dis, true);
                int dominoNumber = 0;
                if (this.remoteVersion == null
                    || (this.remoteVersion.compareTo(Version.GFE_80) >= 0)) {
                  dominoNumber = dis.readInt();
                  if (this.sharedResource) {
                    dominoNumber = 0;
                  }
                  dominoCount.set(dominoNumber);
                  // this.senderName = dis.readUTF();
                }
                if (!this.sharedResource) {
                  if (tipDomino()) {
                    logger.info(LocalizedMessage.create(
                        LocalizedStrings.Connection_THREAD_OWNED_RECEIVER_FORCING_ITSELF_TO_SEND_ON_THREAD_OWNED_SOCKETS));
                    // bug #49565 - if domino count is >= 2 use shared resources.
                    // Also see DistributedCacheOperation#supportsDirectAck
                  } else { // if (dominoNumber < 2) {
                    ConnectionTable.threadWantsOwnResources();
                    if (logger.isDebugEnabled()) {
                      logger.debug(
                          "thread-owned receiver with domino count of {} will prefer sending on thread-owned sockets",
                          dominoNumber);
                    }
                    // } else {
                    // ConnectionTable.threadWantsSharedResources();
                  }
                  this.conduit.getStats().incThreadOwnedReceivers(1L, dominoNumber);
                  // Because this thread is not shared resource, it will be used for direct
                  // ack. Direct ack messages can be large. This call will resize the send
                  // buffer.
                  setSendBufferSize(this.socket);
                }
                // String name = owner.getDM().getConfig().getName();
                // if (name == null) {
                // name = "pid="+OSProcess.getId();
                // }
                setThreadName(dominoNumber);
              } catch (Exception e) {
                this.owner.getConduit().getCancelCriterion().checkCancelInProgress(e); // bug 37101
                logger.fatal(LocalizedMessage.create(
                    LocalizedStrings.Connection_ERROR_DESERIALIZING_P2P_HANDSHAKE_MESSAGE), e);
                this.readerShuttingDown = true;
                requestClose(LocalizedStrings.Connection_ERROR_DESERIALIZING_P2P_HANDSHAKE_MESSAGE
                    .toLocalizedString());
                return;
              }
              if (logger.isDebugEnabled()) {
                logger.debug("P2P handshake remoteAddr is {}{}", this.remoteAddr,
                    (this.remoteVersion != null ? " (" + this.remoteVersion + ')' : ""));
              }
              try {
                String authInit = System.getProperty(
                    DistributionConfigImpl.SECURITY_SYSTEM_PREFIX + SECURITY_PEER_AUTH_INIT);
                boolean isSecure = authInit != null && authInit.length() != 0;

                if (isSecure) {
                  if (owner.getConduit().waitForMembershipCheck(this.remoteAddr)) {
                    sendOKHandshakeReply(); // fix for bug 33224
                    notifyHandshakeWaiter(true);
                  } else {
                    // ARB: check if we need notifyHandshakeWaiter() call.
                    notifyHandshakeWaiter(false);
                    logger.warn(LocalizedMessage.create(
                        LocalizedStrings.Connection_0_TIMED_OUT_DURING_A_MEMBERSHIP_CHECK,
                        p2pReaderName()));
                    return;
                  }
                } else {
                  sendOKHandshakeReply(); // fix for bug 33224
                  try {
                    notifyHandshakeWaiter(true);
                  } catch (Exception e) {
                    logger.fatal(LocalizedMessage
                        .create(LocalizedStrings.Connection_UNCAUGHT_EXCEPTION_FROM_LISTENER), e);
                  }
                }
              } catch (IOException ex) {
                final String err =
                    LocalizedStrings.Connection_FAILED_SENDING_HANDSHAKE_REPLY.toLocalizedString();
                if (logger.isDebugEnabled()) {
                  logger.debug(err, ex);
                }
                this.readerShuttingDown = true;
                requestClose(err + ": " + ex);
                return;
              }
            }
          }
          if (!connected) {
            continue;
          }
          accessed();
          nioInputBuffer.limit(oldLimit);
          nioInputBuffer.position(startPos + nioMessageLength);
        } else {
          done = true;
          compactOrResizeBuffer(nioMessageLength);
        }
      } else {
        done = true;
        if (nioInputBuffer.position() != 0) {
          nioInputBuffer.compact();
        } else {
          nioInputBuffer.position(nioInputBuffer.limit());
          nioInputBuffer.limit(nioInputBuffer.capacity());
        }
      }
    }
  }

  private void setThreadName(int dominoNumber) {
    Thread.currentThread().setName("P2P message reader for " + this.remoteAddr + " "
        + (this.sharedResource ? "" : "un") + "shared" + " " + (this.preserveOrder ? "" : "un")
        + "ordered" + " uid=" + this.uniqueId + (dominoNumber > 0 ? (" dom #" + dominoNumber) : "")
        + " port=" + this.socket.getPort());
  }

  private void compactOrResizeBuffer(int messageLength) {
    final int oldBufferSize = nioInputBuffer.capacity();
    final DMStats stats = this.owner.getConduit().getStats();
    int allocSize = messageLength + MSG_HEADER_BYTES;
    if (oldBufferSize < allocSize) {
      // need a bigger buffer
      logger.info(LocalizedMessage.create(
          LocalizedStrings.Connection_ALLOCATING_LARGER_NETWORK_READ_BUFFER_NEW_SIZE_IS_0_OLD_SIZE_WAS_1,
          new Object[] {Integer.valueOf(allocSize), Integer.valueOf(oldBufferSize)}));
      ByteBuffer oldBuffer = nioInputBuffer;
      nioInputBuffer = Buffers.acquireReceiveBuffer(allocSize, stats);

      if (oldBuffer != null) {
        int oldByteCount = oldBuffer.remaining();
        nioInputBuffer.put(oldBuffer);
        nioInputBuffer.position(oldByteCount);
        Buffers.releaseReceiveBuffer(oldBuffer, stats);
      }
    } else {
      if (nioInputBuffer.position() != 0) {
        nioInputBuffer.compact();
      } else {
        nioInputBuffer.position(nioInputBuffer.limit());
        nioInputBuffer.limit(nioInputBuffer.capacity());
      }
    }
  }

  private boolean dispatchMessage(DistributionMessage msg, int bytesRead, boolean directAck) {
    try {
      msg.setDoDecMessagesBeingReceived(true);
      if (directAck) {
        Assert.assertTrue(!isSharedResource(),
            "We were asked to send a direct reply on a shared socket");
        msg.setReplySender(new DirectReplySender(this));
      }
      this.owner.getConduit().messageReceived(this, msg, bytesRead);
      return true;
    } finally {
      if (msg.containsRegionContentChange()) {
        messagesReceived++;
      }
    }
  }

  protected TCPConduit getConduit() {
    return this.conduit;
  }

  protected Socket getSocket() throws SocketException {
    // fix for bug 37286
    Socket result = this.socket;
    if (result == null) {
      throw new SocketException(
          LocalizedStrings.Connection_SOCKET_HAS_BEEN_CLOSED.toLocalizedString());
    }
    return result;
  }

  public boolean isSocketClosed() {
    return this.socket.isClosed() || !this.socket.isConnected();
  }

  public boolean isReceiverStopped() {
    return this.stopped;
  }

  private boolean isSocketInUse() {
    return this.socketInUse;
  }


  protected void accessed() {
    this.accessed = true;
  }

  /**
   * return the DM id of the guy on the other side of this connection.
   */
  public InternalDistributedMember getRemoteAddress() {
    return this.remoteAddr;
  }

  /**
   * Return the version of the guy on the other side of this connection.
   */
  public Version getRemoteVersion() {
    return this.remoteVersion;
  }

  @Override
  public String toString() {
    return String.valueOf(remoteAddr) + '@' + this.uniqueId
        + (this.remoteVersion != null ? ('(' + this.remoteVersion.toString() + ')')
            : "") /*
                   * DEBUG + " accepted=" + this.isReceiver + " connected=" + this.connected +
                   * " hash=" + System.identityHashCode(this) + " preserveOrder=" +
                   * this.preserveOrder + " closing=" + isClosing() + ">"
                   */;
  }

  /**
   * answers whether this connection was initiated in this vm
   *
   * @return true if the connection was initiated here
   * @since GemFire 5.1
   */
  protected boolean getOriginatedHere() {
    return !this.isReceiver;
  }

  /**
   * answers whether this connection is used for ordered message delivery
   */
  protected boolean getPreserveOrder() {
    return preserveOrder;
  }

  /**
   * answers the unique ID of this connection in the originating VM
   */
  protected long getUniqueId() {
    return this.uniqueId;
  }

  /**
   * answers the number of messages received by this connection
   */
  protected long getMessagesReceived() {
    return messagesReceived;
  }

  /**
   * answers the number of messages sent on this connection
   */
  protected long getMessagesSent() {
    return messagesSent;
  }

  public void acquireSendPermission() throws ConnectionException {
    if (!this.connected) {
      throw new ConnectionException(
          LocalizedStrings.Connection_CONNECTION_IS_CLOSED.toLocalizedString());
    }
    if (isReaderThread()) {
      // reader threads send replies and we always want to permit those without waiting
      return;
    }
    boolean interrupted = false;
    try {
      for (;;) {
        this.owner.getConduit().getCancelCriterion().checkCancelInProgress(null);
        try {
          this.senderSem.acquire();
          break;
        } catch (InterruptedException ex) {
          interrupted = true;
        }
      } // for
    } finally {
      if (interrupted) {
        Thread.currentThread().interrupt();
      }
    }
    if (!this.connected) {
      this.senderSem.release();
      this.owner.getConduit().getCancelCriterion().checkCancelInProgress(null); // bug 37101
      throw new ConnectionException(
          LocalizedStrings.Connection_CONNECTION_IS_CLOSED.toLocalizedString());
    }
  }

  public void releaseSendPermission() {
    if (isReaderThread()) {
      return;
    }
    this.senderSem.release();
  }

  private void closeSenderSem() {
    // All we need to do is increase the number of permits by one
    // just in case 1 or more guys are currently waiting to acquire.
    // One of them will get it and then find out the connection is closed
    // and then he will release it until all guys currently waiting to acquire
    // will complete by throwing a ConnectionException.
    releaseSendPermission();
  }

  boolean nioChecked;
  boolean useNIO;

  private boolean useNIO() {
    if (TCPConduit.useSSL) {
      return false;
    }
    if (this.nioChecked) {
      return this.useNIO;
    }
    this.nioChecked = true;
    this.useNIO = this.owner.getConduit().useNIO();
    if (!this.useNIO) {
      return false;
    }
    // JDK bug 6230761 - NIO can't be used with IPv6 on Windows
    if (this.socket != null && (this.socket.getInetAddress() instanceof Inet6Address)) {
      String os = System.getProperty("os.name");
      if (os != null) {
        if (os.contains("Windows")) {
          this.useNIO = false;
        }
      }
    }
    return this.useNIO;
  }
}
