Add buffer sizes to the UDP sockets

git-svn-id: https://svn.apache.org/repos/asf/tomcat/trunk@629539 13f79535-47bb-0310-9956-ffa450edef68

diff --git a/java/org/apache/catalina/tribes/membership/McastServiceImpl.java b/java/org/apache/catalina/tribes/membership/McastServiceImpl.java
index fd410bd..1c4341a 100644 (file)
--- a/java/org/apache/catalina/tribes/membership/McastServiceImpl.java
+++ b/java/org/apache/catalina/tribes/membership/McastServiceImpl.java
@@ -421,12 +421,15 @@ public class McastServiceImpl
                     if ( log.isDebugEnabled() )
                         log.debug("Invalid member mcast package.",ax);
                 } catch ( Exception x ) {
-                    if (errorCounter==0) log.warn("Error receiving mcast package. Sleeping 500ms",x);
-                    else log.debug("Error receiving mcast package. Sleeping 500ms",x);
-                    try { Thread.sleep(500); } catch ( Exception ignore ){}
-                    if ( (++errorCounter)>=recoveryCounter ) {
-                        errorCounter=0;
-                        new RecoveryThread(McastServiceImpl.this);
+                    if (x instanceof InterruptedException) interrupted();
+                    else {
+                        if (errorCounter==0) log.warn("Error receiving mcast package. Sleeping 500ms",x);
+                        else log.debug("Error receiving mcast package. Sleeping 500ms",x);
+                        try { Thread.sleep(500); } catch ( Exception ignore ){}
+                        if ( (++errorCounter)>=recoveryCounter ) {
+                            errorCounter=0;
+                            new RecoveryThread(McastServiceImpl.this);
+                        }
                     }
                 }
             }

diff --git a/java/org/apache/catalina/tribes/transport/AbstractSender.java b/java/org/apache/catalina/tribes/transport/AbstractSender.java
index aa3a2d0..258dbda 100644 (file)
--- a/java/org/apache/catalina/tribes/transport/AbstractSender.java
+++ b/java/org/apache/catalina/tribes/transport/AbstractSender.java
@@ -38,6 +38,8 @@ public abstract class AbstractSender implements DataSender {
     private boolean connected = false;
     private int rxBufSize = 25188;
     private int txBufSize = 43800;
+    private int udpRxBufSize = 25188;
+    private int udpTxBufSize = 43800;
     private boolean directBuffer = false;
     private int keepAliveCount = -1;
     private int requestCount = 0;
@@ -332,4 +334,24 @@ public abstract class AbstractSender implements DataSender {
         this.udpPort = udpPort;
     }
 
+
+    public int getUdpRxBufSize() {
+        return udpRxBufSize;
+    }
+
+
+    public void setUdpRxBufSize(int udpRxBufSize) {
+        this.udpRxBufSize = udpRxBufSize;
+    }
+
+
+    public int getUdpTxBufSize() {
+        return udpTxBufSize;
+    }
+
+
+    public void setUdpTxBufSize(int udpTxBufSize) {
+        this.udpTxBufSize = udpTxBufSize;
+    }
+
 }
\ No newline at end of file

diff --git a/java/org/apache/catalina/tribes/transport/ReceiverBase.java b/java/org/apache/catalina/tribes/transport/ReceiverBase.java
index 06d4217..50c6899 100644 (file)
--- a/java/org/apache/catalina/tribes/transport/ReceiverBase.java
+++ b/java/org/apache/catalina/tribes/transport/ReceiverBase.java
@@ -56,6 +56,9 @@ public abstract class ReceiverBase implements ChannelReceiver, ListenCallback, R
     private int securePort = -1;
     private int rxBufSize = 43800;
     private int txBufSize = 25188;
+    private int udpRxBufSize = 43800;
+    private int udpTxBufSize = 25188;
+
     private boolean listen = false;
     private RxTaskPool pool;
     private boolean direct = true;
@@ -520,4 +523,20 @@ public abstract class ReceiverBase implements ChannelReceiver, ListenCallback, R
         this.udpPort = udpPort;
     }
 
+    public int getUdpRxBufSize() {
+        return udpRxBufSize;
+    }
+
+    public void setUdpRxBufSize(int udpRxBufSize) {
+        this.udpRxBufSize = udpRxBufSize;
+    }
+
+    public int getUdpTxBufSize() {
+        return udpTxBufSize;
+    }
+
+    public void setUdpTxBufSize(int udpTxBufSize) {
+        this.udpTxBufSize = udpTxBufSize;
+    }
+
 }
\ No newline at end of file

diff --git a/java/org/apache/catalina/tribes/transport/nio/NioReceiver.java b/java/org/apache/catalina/tribes/transport/nio/NioReceiver.java
index de52763..0c91f1f 100644 (file)
--- a/java/org/apache/catalina/tribes/transport/nio/NioReceiver.java
+++ b/java/org/apache/catalina/tribes/transport/nio/NioReceiver.java
@@ -250,6 +250,11 @@ public class NioReceiver extends ReceiverBase implements Runnable, ChannelReceiv
         setListen(true);
         if (selector!=null && datagramChannel!=null) {
             ObjectReader oreader = new ObjectReader(MAX_UDP_SIZE); //max size for a datagram packet
+            datagramChannel.socket().setSendBufferSize(getUdpTxBufSize());
+            datagramChannel.socket().setReceiveBufferSize(getUdpRxBufSize());
+            datagramChannel.socket().setReuseAddress(getSoReuseAddress());
+            datagramChannel.socket().setSoTimeout(getTimeout());
+            datagramChannel.socket().setTrafficClass(getSoTrafficClass());
             registerChannel(selector,datagramChannel,SelectionKey.OP_READ,oreader);
         }
 

diff --git a/java/org/apache/catalina/tribes/transport/nio/NioReplicationTask.java b/java/org/apache/catalina/tribes/transport/nio/NioReplicationTask.java
index 6e37070..8cb0dbe 100644 (file)
--- a/java/org/apache/catalina/tribes/transport/nio/NioReplicationTask.java
+++ b/java/org/apache/catalina/tribes/transport/nio/NioReplicationTask.java
@@ -308,8 +308,12 @@ public class NioReplicationTask extends AbstractRxTask {
             int total = 0;
             if (channel instanceof DatagramChannel) {
                 DatagramChannel dchannel = (DatagramChannel)channel;
-                while ( total < command.length ) {
-                    total += dchannel.send(buf, udpaddr);
+                //were using a shared channel, it's not thread safe
+                //TODO check optimization, one channel per thread
+                synchronized (dchannel) {
+                    while ( total < command.length ) {
+                        total += dchannel.send(buf, udpaddr);
+                    }
                 }
             } else {
                 while ( total < command.length ) {

diff --git a/java/org/apache/catalina/tribes/transport/nio/NioSender.java b/java/org/apache/catalina/tribes/transport/nio/NioSender.java
index b8e0479..3db44e4 100644 (file)
--- a/java/org/apache/catalina/tribes/transport/nio/NioSender.java
+++ b/java/org/apache/catalina/tribes/transport/nio/NioSender.java
@@ -149,8 +149,8 @@ public class NioSender extends AbstractSender implements DataSender{
             socketChannel.socket().setSoLinger(getSoLingerOn(),getSoLingerTime());
             socketChannel.socket().setTrafficClass(getSoTrafficClass());
         } else if (dataChannel!=null) {
-            dataChannel.socket().setSendBufferSize(getTxBufSize());
-            dataChannel.socket().setReceiveBufferSize(getRxBufSize());
+            dataChannel.socket().setSendBufferSize(getUdpTxBufSize());
+            dataChannel.socket().setReceiveBufferSize(getUdpRxBufSize());
             dataChannel.socket().setSoTimeout((int)getTimeout());
             dataChannel.socket().setReuseAddress(getSoReuseAddress());
             dataChannel.socket().setTrafficClass(getSoTrafficClass());

diff --git a/test/org/apache/catalina/tribes/test/channel/TestUdpPackages.java b/test/org/apache/catalina/tribes/test/channel/TestUdpPackages.java
index c159d28..dc4f39a 100644 (file)
--- a/test/org/apache/catalina/tribes/test/channel/TestUdpPackages.java
+++ b/test/org/apache/catalina/tribes/test/channel/TestUdpPackages.java
@@ -20,6 +20,8 @@ import junit.framework.TestCase;
 import java.io.Serializable;\r
 import java.util.Random;\r
 import java.util.Arrays;\r
+import java.util.concurrent.atomic.AtomicInteger;\r
+import java.util.concurrent.atomic.AtomicLong;\r
 \r
 import org.apache.catalina.tribes.Channel;\r
 import org.apache.catalina.tribes.ChannelListener;\r
@@ -33,6 +35,7 @@ import org.apache.catalina.tribes.transport.ReplicationTransmitter;
 import org.apache.catalina.tribes.group.interceptors.MessageDispatchInterceptor;\r
 import org.apache.catalina.tribes.group.interceptors.MessageDispatch15Interceptor;\r
 import org.apache.catalina.tribes.group.interceptors.ThroughputInterceptor;\r
+import org.apache.catalina.tribes.io.XByteBuffer;\r
 \r
 /**\r
  */\r
@@ -81,11 +84,18 @@ public class TestUdpPackages extends TestCase {
         channel1.send(new Member[] {channel2.getLocalMember(false)}, Data.createRandomData(1024),Channel.SEND_OPTIONS_UDP);\r
         Thread.sleep(500);\r
         System.err.println("Finished Single package NO_ACK ["+listener1.count+"]");\r
-        assertEquals("Checking success messages.",1,listener1.count);\r
+        assertEquals("Checking success messages.",1,listener1.count.get());\r
     }\r
 \r
     \r
     public void testDataSendNO_ACK() throws Exception {\r
+        final AtomicInteger counter = new AtomicInteger(0);\r
+        ReceiverBase rb1 = (ReceiverBase)channel1.getChannelReceiver();\r
+        ReceiverBase rb2 = (ReceiverBase)channel2.getChannelReceiver();\r
+        rb1.setUdpRxBufSize(1024*1024*10);\r
+        rb2.setUdpRxBufSize(1024*1024*10);\r
+        rb1.setUdpTxBufSize(1024*1024*10);\r
+        rb2.setUdpTxBufSize(1024*1024*10);\r
         System.err.println("Starting NO_ACK");\r
         Thread[] threads = new Thread[threadCount];\r
         for (int x=0; x<threads.length; x++ ) {\r
@@ -93,7 +103,11 @@ public class TestUdpPackages extends TestCase {
                 public void run() {\r
                     try {\r
                         long start = System.currentTimeMillis();\r
-                        for (int i = 0; i < msgCount; i++) channel1.send(new Member[] {channel2.getLocalMember(false)}, Data.createRandomData(1024),Channel.SEND_OPTIONS_UDP);\r
+                        for (int i = 0; i < msgCount; i++) {\r
+                            int cnt = counter.addAndGet(1);\r
+                            channel1.send(new Member[] {channel2.getLocalMember(false)}, Data.createRandomData(1024,cnt),Channel.SEND_OPTIONS_UDP);\r
+                            Thread.currentThread().sleep(10);\r
+                        }\r
                         System.out.println("Thread["+this.getName()+"] sent "+msgCount+" messages in "+(System.currentTimeMillis()-start)+" ms.");\r
                     }catch ( Exception x ) {\r
                         x.printStackTrace();\r
@@ -108,9 +122,19 @@ public class TestUdpPackages extends TestCase {
         for (int x=0; x<threads.length; x++ ) { threads[x].join();}\r
         //sleep for 50 sec, let the other messages in\r
         long start = System.currentTimeMillis();\r
-        while ( (System.currentTimeMillis()-start)<25000 && msgCount*threadCount!=listener1.count) Thread.sleep(500);\r
+        while ( (System.currentTimeMillis()-start)<25000 && msgCount*threadCount!=listener1.count.get()) Thread.sleep(500);\r
         System.err.println("Finished NO_ACK ["+listener1.count+"]");\r
-        assertEquals("Checking success messages.",msgCount*threadCount,listener1.count);\r
+        System.out.println("Sent "+counter.get()+ " messages. Received "+listener1.count+" Highest msg received:"+listener1.maxIdx);\r
+        System.out.print("Missing messages:");\r
+        printMissingMsgs(listener1.nrs,counter.get());\r
+        assertEquals("Checking success messages.",msgCount*threadCount,listener1.count.get());\r
+    }\r
+    \r
+    public static void printMissingMsgs(int[] msgs, int maxIdx) {\r
+        for (int i=0; i<maxIdx && i<msgs.length; i++) {\r
+            if (msgs[i]==0) System.out.print(i+", ");\r
+        }\r
+        System.out.println();\r
     }\r
 \r
     public void testDataSendASYNCM() throws Exception {\r
@@ -136,18 +160,18 @@ public class TestUdpPackages extends TestCase {
             for (int x=0; x<threads.length; x++ ) { threads[x].join();}\r
             //sleep for 50 sec, let the other messages in\r
             long start = System.currentTimeMillis();\r
-            while ( (System.currentTimeMillis()-start)<25000 && msgCount*threadCount!=listener1.count) Thread.sleep(500);\r
+            while ( (System.currentTimeMillis()-start)<25000 && msgCount*threadCount!=listener1.count.get()) Thread.sleep(500);\r
             System.err.println("Finished ASYNC MULTI THREAD ["+listener1.count+"]");\r
-            assertEquals("Checking success messages.",msgCount*threadCount,listener1.count);\r
+            assertEquals("Checking success messages.",msgCount*threadCount,listener1.count.get());\r
     }\r
     public void testDataSendASYNC() throws Exception {\r
         System.err.println("Starting ASYNC");\r
         for (int i=0; i<msgCount; i++) channel1.send(new Member[] {channel2.getLocalMember(false)},Data.createRandomData(1024),GroupChannel.SEND_OPTIONS_ASYNCHRONOUS|Channel.SEND_OPTIONS_UDP);\r
         //sleep for 50 sec, let the other messages in\r
         long start = System.currentTimeMillis();\r
-        while ( (System.currentTimeMillis()-start)<5000 && msgCount!=listener1.count) Thread.sleep(500);\r
+        while ( (System.currentTimeMillis()-start)<5000 && msgCount!=listener1.count.get()) Thread.sleep(500);\r
         System.err.println("Finished ASYNC");\r
-        assertEquals("Checking success messages.",msgCount,listener1.count);\r
+        assertEquals("Checking success messages.",msgCount,listener1.count.get());\r
     }\r
 \r
     public void testDataSendACK() throws Exception {\r
@@ -155,7 +179,7 @@ public class TestUdpPackages extends TestCase {
         for (int i=0; i<msgCount; i++) channel1.send(new Member[] {channel2.getLocalMember(false)},Data.createRandomData(1024),GroupChannel.SEND_OPTIONS_USE_ACK|Channel.SEND_OPTIONS_UDP);\r
         Thread.sleep(250);\r
         System.err.println("Finished ACK");\r
-        assertEquals("Checking success messages.",msgCount,listener1.count);\r
+        assertEquals("Checking success messages.",msgCount,listener1.count.get());\r
     }\r
 \r
     public void testDataSendSYNCACK() throws Exception {\r
@@ -163,24 +187,38 @@ public class TestUdpPackages extends TestCase {
         for (int i=0; i<msgCount; i++) channel1.send(new Member[] {channel2.getLocalMember(false)},Data.createRandomData(1024),GroupChannel.SEND_OPTIONS_SYNCHRONIZED_ACK|GroupChannel.SEND_OPTIONS_USE_ACK|Channel.SEND_OPTIONS_UDP);\r
         Thread.sleep(250);\r
         System.err.println("Finished SYNC_ACK");\r
-        assertEquals("Checking success messages.",msgCount,listener1.count);\r
+        assertEquals("Checking success messages.",msgCount,listener1.count.get());\r
     }\r
 \r
     public static class Listener implements ChannelListener {\r
-        long count = 0;\r
+        AtomicLong count = new AtomicLong(0);\r
+        int maxIdx = -1;\r
+        int[] nrs = new int[1000000];\r
+        public Listener() {\r
+            Arrays.fill(nrs, 0);\r
+        }\r
         public boolean accept(Serializable s, Member m) {\r
             return (s instanceof Data);\r
         }\r
 \r
         public void messageReceived(Serializable s, Member m) {\r
-            Data d = (Data)s;\r
-            if ( !Data.verify(d) ) {\r
-                System.err.println("ERROR");\r
-            } else {\r
-                count++;\r
-                if ((count %1000) ==0 ) {\r
-                    System.err.println("SUCCESS:"+count);\r
+            try {\r
+                Data d = (Data)s;\r
+                if ( !Data.verify(d) ) {\r
+                    System.err.println("ERROR - Unable to verify data package");\r
+                } else {\r
+                    long c = count.addAndGet(1);\r
+                    if ((c%1000) ==0 ) {\r
+                        System.err.println("SUCCESS:"+c);\r
+                    }\r
+                    int nr = d.getNumber();\r
+                    if (nr>=0 && nr<nrs.length) {\r
+                        maxIdx = Math.max(maxIdx, nr);\r
+                        nrs[nr] = 1;\r
+                    }\r
                 }\r
+            }catch (Exception x ) {\r
+                x.printStackTrace();\r
             }\r
         }\r
     }\r
@@ -189,25 +227,41 @@ public class TestUdpPackages extends TestCase {
         public int length;\r
         public byte[] data;\r
         public byte key;\r
+        public boolean hasNr = false;\r
         public static Random r = new Random(System.currentTimeMillis());\r
         public static Data createRandomData() {\r
             return createRandomData(ChannelReceiver.MAX_UDP_SIZE);\r
         }\r
         public static Data createRandomData(int size) {\r
+            return createRandomData(size,-1);\r
+        }\r
+        \r
+        public static Data createRandomData(int size, int number) {\r
             int i = r.nextInt();\r
             i = ( i % 127 );\r
             int length = Math.abs(r.nextInt() % size);\r
+            if (length<100) length += 100;\r
             Data d = new Data();\r
             d.length = length;\r
             d.key = (byte)i;\r
             d.data = new byte[length];\r
             Arrays.fill(d.data,d.key);\r
+            if (number>0 && d.data.length>=4) {\r
+                //populate number\r
+                d.hasNr = true;\r
+                XByteBuffer.toBytes(number,d.data, 0);\r
+            }\r
             return d;\r
         }\r
+        \r
+        public int getNumber() {\r
+            if (!hasNr) return -1;\r
+            return XByteBuffer.toInt(this.data, 0);\r
+        }\r
 \r
         public static boolean verify(Data d) {\r
             boolean result = (d.length == d.data.length);\r
-            for ( int i=0; result && (i<d.data.length); i++ ) result = result && d.data[i] == d.key;\r
+            for ( int i=(d.hasNr?4:0); result && (i<d.data.length); i++ ) result = result && d.data[i] == d.key;\r
             return result;\r
         }\r
     }\r

diff --git a/webapps/docs/config/cluster-receiver.xml b/webapps/docs/config/cluster-receiver.xml
index e272910..007b2df 100644 (file)
--- a/webapps/docs/config/cluster-receiver.xml
+++ b/webapps/docs/config/cluster-receiver.xml
@@ -117,6 +117,14 @@
     <attribute name="txBufSize" required="false">
       The sending buffer size on the receiving sockets. Value is in bytes, the default value is <code>25188</code> bytes.
     </attribute>
+    <attribute name="udpRxBufSize" required="false">
+        The receive buffer size on the datagram socket.
+        Default value is <code>25188</code> bytes.
+    </attribute>
+    <attribute name="udpTxBufSize" required="false">
+       The send buffer size on the datagram socket.
+       Default value is <code>43800</code> bytes.
+    </attribute>
     <attribute name="soKeepAlive" required="false">
       Boolean value for the socket SO_KEEPALIVE option. Possible values are <code>true</code> or <code>false</code>.
     </attribute>

diff --git a/webapps/docs/config/cluster-sender.xml b/webapps/docs/config/cluster-sender.xml
index 5485b94..fe82c98 100644 (file)
--- a/webapps/docs/config/cluster-sender.xml
+++ b/webapps/docs/config/cluster-sender.xml
@@ -90,6 +90,14 @@
        The send buffer size on the socket.
        Default value is <code>43800</code> bytes.
       </attribute>
+      <attribute name="udpRxBufSize" required="false">
+        The receive buffer size on the datagram socket.
+        Default value is <code>25188</code> bytes.
+      </attribute>
+      <attribute name="udpTxBufSize" required="false">
+       The send buffer size on the datagram socket.
+       Default value is <code>43800</code> bytes.
+      </attribute>
       <attribute name="direct" required="false">
        Possible values are <code>true</code> or <code>false</code>. 
        Set to true if you want the receiver to use direct bytebuffers when reading data