001/**
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.activemq.partition;
018
019import org.apache.activemq.broker.*;
020import org.apache.activemq.command.*;
021import org.apache.activemq.partition.dto.Partitioning;
022import org.apache.activemq.partition.dto.Target;
023import org.apache.activemq.state.ConsumerState;
024import org.apache.activemq.state.SessionState;
025import org.apache.activemq.transport.Transport;
026import org.apache.activemq.util.LRUCache;
027import org.slf4j.Logger;
028import org.slf4j.LoggerFactory;
029
030import java.net.InetSocketAddress;
031import java.net.Socket;
032import java.net.SocketAddress;
033import java.util.*;
034import java.util.concurrent.ConcurrentHashMap;
035
036/**
037 * A BrokerFilter which partitions client connections over a cluster of brokers.
038 *
039 * It can use a client identifier like client id, authenticated user name, source ip
040 * address or even destination being used by the connection to figure out which
041 * is the best broker in the cluster that the connection should be using and then
042 * redirects failover clients to that broker.
043 */
044public class PartitionBroker extends BrokerFilter {
045
046    protected static final Logger LOG = LoggerFactory.getLogger(PartitionBroker.class);
047    protected final PartitionBrokerPlugin plugin;
048    protected boolean reloadConfigOnPoll = true;
049
050    public PartitionBroker(Broker broker, PartitionBrokerPlugin plugin) {
051        super(broker);
052        this.plugin = plugin;
053    }
054
055    @Override
056    public void start() throws Exception {
057        super.start();
058        getExecutor().execute(new Runnable() {
059            @Override
060            public void run() {
061                Thread.currentThread().setName("Partition Monitor");
062                onMonitorStart();
063                try {
064                    runPartitionMonitor();
065                } catch (Exception e) {
066                    onMonitorStop();
067                }
068            }
069        });
070    }
071
072    protected void onMonitorStart() {
073    }
074    protected void onMonitorStop() {
075    }
076
077    protected void runPartitionMonitor() {
078        while( !isStopped() ) {
079            try {
080                monitorWait();
081            } catch (InterruptedException e) {
082                break;
083            }
084
085            if(reloadConfigOnPoll) {
086                try {
087                    reloadConfiguration();
088                } catch (Exception e) {
089                    continue;
090                }
091            }
092
093            for( ConnectionMonitor monitor: monitors.values()) {
094                checkTarget(monitor);
095            }
096        }
097    }
098
099    protected void monitorWait() throws InterruptedException {
100        synchronized (this) {
101            this.wait(1000);
102        }
103    }
104
105    protected void monitorWakeup()  {
106        synchronized (this) {
107            this.notifyAll();
108        }
109    }
110
111    protected void reloadConfiguration() throws Exception {
112    }
113
114    protected void checkTarget(ConnectionMonitor monitor) {
115
116        // can we find a preferred target for the connection?
117        Target targetDTO = pickBestBroker(monitor);
118        if( targetDTO == null || targetDTO.ids==null) {
119            LOG.debug("No partition target found for connection: "+monitor.context.getConnectionId());
120            return;
121        }
122
123        // Are we one the the targets?
124        if( targetDTO.ids.contains(getBrokerName()) ) {
125            LOG.debug("We are a partition target for connection: "+monitor.context.getConnectionId());
126            return;
127        }
128
129        // Then we need to move the connection over.
130        String connectionString = getConnectionString(targetDTO.ids);
131        if( connectionString==null ) {
132            LOG.debug("Could not convert to partition targets to connection string: " + targetDTO.ids);
133            return;
134        }
135
136        LOG.info("Redirecting connection to: " + connectionString);
137        TransportConnection connection = (TransportConnection)monitor.context.getConnection();
138        ConnectionControl cc = new ConnectionControl();
139        cc.setConnectedBrokers(connectionString);
140        cc.setRebalanceConnection(true);
141        connection.dispatchAsync(cc);
142    }
143
144    protected String getConnectionString(HashSet<String> ids) {
145        StringBuilder rc = new StringBuilder();
146        for (String id : ids) {
147            String url = plugin.getBrokerURL(this, id);
148            if( url!=null ) {
149                if( rc.length()!=0 ) {
150                    rc.append(',');
151                }
152                rc.append(url);
153            }
154        }
155        if( rc.length()==0 )
156            return null;
157        return rc.toString();
158    }
159
160    static private class Score {
161        int value;
162    }
163
164    protected Target pickBestBroker(ConnectionMonitor monitor) {
165
166        if( getConfig() ==null )
167            return null;
168
169        if( getConfig().bySourceIp !=null && !getConfig().bySourceIp.isEmpty() ) {
170            TransportConnection connection = (TransportConnection)monitor.context.getConnection();
171            Transport transport = connection.getTransport();
172            Socket socket = transport.narrow(Socket.class);
173            if( socket !=null ) {
174                SocketAddress address = socket.getRemoteSocketAddress();
175                if( address instanceof InetSocketAddress) {
176                    String ip = ((InetSocketAddress) address).getAddress().getHostAddress();
177                    Target targetDTO = getConfig().bySourceIp.get(ip);
178                    if( targetDTO!=null ) {
179                        return targetDTO;
180                    }
181                }
182            }
183        }
184
185        if( getConfig().byUserName !=null && !getConfig().byUserName.isEmpty() ) {
186            String userName = monitor.context.getUserName();
187            if( userName !=null ) {
188                Target targetDTO = getConfig().byUserName.get(userName);
189                if( targetDTO!=null ) {
190                    return targetDTO;
191                }
192            }
193        }
194
195        if( getConfig().byClientId !=null && !getConfig().byClientId.isEmpty() ) {
196            String clientId = monitor.context.getClientId();
197            if( clientId!=null ) {
198                Target targetDTO = getConfig().byClientId.get(clientId);
199                if( targetDTO!=null ) {
200                    return targetDTO;
201                }
202            }
203        }
204
205        if(
206             (getConfig().byQueue !=null && !getConfig().byQueue.isEmpty())
207          || (getConfig().byTopic !=null && !getConfig().byTopic.isEmpty())
208          ) {
209
210            // Collect the destinations the connection is consuming from...
211            HashSet<ActiveMQDestination> dests = new HashSet<ActiveMQDestination>();
212            for (SessionState session : monitor.context.getConnectionState().getSessionStates()) {
213                for (ConsumerState consumer : session.getConsumerStates()) {
214                    ActiveMQDestination destination = consumer.getInfo().getDestination();
215                    if( destination.isComposite() ) {
216                        dests.addAll(Arrays.asList(destination.getCompositeDestinations()));
217                    } else {
218                        dests.addAll(Collections.singletonList(destination));
219                    }
220                }
221            }
222
223            // Group them by the partitioning target for the destinations and score them..
224            HashMap<Target, Score> targetScores = new HashMap<Target, Score>();
225            for (ActiveMQDestination dest : dests) {
226                Target target = getTarget(dest);
227                if( target!=null ) {
228                    Score score = targetScores.get(target);
229                    if( score == null ) {
230                        score = new Score();
231                        targetScores.put(target, score);
232                    }
233                    score.value++;
234                }
235            }
236
237            // The target with largest score wins..
238            if( !targetScores.isEmpty() ) {
239                Target bestTarget = null;
240                int bestScore=0;
241                for (Map.Entry<Target, Score> entry : targetScores.entrySet()) {
242                    if( entry.getValue().value > bestScore ) {
243                        bestTarget = entry.getKey();
244                    }
245                }
246                return bestTarget;
247            }
248
249            // If we get here is because there were no consumers, or the destinations for those
250            // consumers did not have an assigned destination..  So partition based on producer
251            // usage.
252            Target best = monitor.findBestProducerTarget(this);
253            if( best!=null ) {
254                return best;
255            }
256        }
257        return null;
258    }
259
260    protected Target getTarget(ActiveMQDestination dest) {
261        Partitioning config = getConfig();
262        if( dest.isQueue() && config.byQueue !=null && !config.byQueue.isEmpty() ) {
263            return config.byQueue.get(dest.getPhysicalName());
264        } else if( dest.isTopic() && config.byTopic !=null && !config.byTopic.isEmpty() ) {
265            return config.byTopic.get(dest.getPhysicalName());
266        }
267        return null;
268    }
269
270    protected final ConcurrentHashMap<ConnectionId, ConnectionMonitor> monitors = new ConcurrentHashMap<ConnectionId, ConnectionMonitor>();
271
272    @Override
273    public void addConnection(ConnectionContext context, ConnectionInfo info) throws Exception {
274        if( info.isFaultTolerant() ) {
275            ConnectionMonitor monitor = new ConnectionMonitor(context);
276            monitors.put(info.getConnectionId(), monitor);
277            super.addConnection(context, info);
278            checkTarget(monitor);
279        } else {
280            super.addConnection(context, info);
281        }
282    }
283
284    @Override
285    public void removeConnection(ConnectionContext context, ConnectionInfo info, Throwable error) throws Exception {
286        super.removeConnection(context, info, error);
287        if( info.isFaultTolerant() ) {
288            monitors.remove(info.getConnectionId());
289        }
290    }
291
292    @Override
293    public void send(ProducerBrokerExchange producerExchange, Message messageSend) throws Exception {
294        ConnectionMonitor monitor = monitors.get(producerExchange.getConnectionContext().getConnectionId());
295        if( monitor!=null ) {
296            monitor.onSend(producerExchange, messageSend);
297        }
298    }
299
300    protected Partitioning getConfig() {
301        return plugin.getConfig();
302    }
303
304
305    static class Traffic {
306        long messages;
307        long bytes;
308    }
309
310    static class ConnectionMonitor {
311
312        final ConnectionContext context;
313        LRUCache<ActiveMQDestination, Traffic> trafficPerDestination =  new LRUCache<ActiveMQDestination, Traffic>();
314
315        public ConnectionMonitor(ConnectionContext context) {
316            this.context = context;
317        }
318
319        synchronized public Target findBestProducerTarget(PartitionBroker broker) {
320            Target best = null;
321            long bestSize = 0 ;
322            for (Map.Entry<ActiveMQDestination, Traffic> entry : trafficPerDestination.entrySet()) {
323                Traffic t = entry.getValue();
324                // Once we get enough messages...
325                if( t.messages < broker.plugin.getMinTransferCount()) {
326                    continue;
327                }
328                if( t.bytes > bestSize) {
329                    bestSize = t.bytes;
330                    Target target = broker.getTarget(entry.getKey());
331                    if( target!=null ) {
332                        best = target;
333                    }
334                }
335            }
336            return best;
337        }
338
339        synchronized public void onSend(ProducerBrokerExchange producerExchange, Message message) {
340            ActiveMQDestination dest = message.getDestination();
341            Traffic traffic = trafficPerDestination.get(dest);
342            if( traffic == null ) {
343                traffic = new Traffic();
344                trafficPerDestination.put(dest, traffic);
345            }
346            traffic.messages += 1;
347            traffic.bytes += message.getSize();
348        }
349
350
351    }
352
353}