From: fhanik <fhanik@13f79535-47bb-0310-9956-ffa450edef68>
Date: Wed, 20 Feb 2008 17:12:27 +0000 (+0000)
Subject: Add buffer sizes to the UDP sockets
X-Git-Url: https://git.internetallee.de/?a=commitdiff_plain;h=af36faf48654cc1208673c5ad520ae06ef7e1c05;p=tomcat7.0

Add buffer sizes to the UDP sockets


git-svn-id: https://svn.apache.org/repos/asf/tomcat/trunk@629539 13f79535-47bb-0310-9956-ffa450edef68
---

diff --git a/java/org/apache/catalina/tribes/membership/McastServiceImpl.java b/java/org/apache/catalina/tribes/membership/McastServiceImpl.java
index fd410bde7..1c4341a52 100644
--- a/java/org/apache/catalina/tribes/membership/McastServiceImpl.java
+++ b/java/org/apache/catalina/tribes/membership/McastServiceImpl.java
@@ -421,12 +421,15 @@ public class McastServiceImpl
                     if ( log.isDebugEnabled() )
                         log.debug("Invalid member mcast package.",ax);
                 } catch ( Exception x ) {
-                    if (errorCounter==0) log.warn("Error receiving mcast package. Sleeping 500ms",x);
-                    else log.debug("Error receiving mcast package. Sleeping 500ms",x);
-                    try { Thread.sleep(500); } catch ( Exception ignore ){}
-                    if ( (++errorCounter)>=recoveryCounter ) {
-                        errorCounter=0;
-                        new RecoveryThread(McastServiceImpl.this);
+                    if (x instanceof InterruptedException) interrupted();
+                    else {
+                        if (errorCounter==0) log.warn("Error receiving mcast package. Sleeping 500ms",x);
+                        else log.debug("Error receiving mcast package. Sleeping 500ms",x);
+                        try { Thread.sleep(500); } catch ( Exception ignore ){}
+                        if ( (++errorCounter)>=recoveryCounter ) {
+                            errorCounter=0;
+                            new RecoveryThread(McastServiceImpl.this);
+                        }
                     }
                 }
             }
diff --git a/java/org/apache/catalina/tribes/transport/AbstractSender.java b/java/org/apache/catalina/tribes/transport/AbstractSender.java
index aa3a2d059..258dbdaee 100644
--- a/java/org/apache/catalina/tribes/transport/AbstractSender.java
+++ b/java/org/apache/catalina/tribes/transport/AbstractSender.java
@@ -38,6 +38,8 @@ public abstract class AbstractSender implements DataSender {
     private boolean connected = false;
     private int rxBufSize = 25188;
     private int txBufSize = 43800;
+    private int udpRxBufSize = 25188;
+    private int udpTxBufSize = 43800;
     private boolean directBuffer = false;
     private int keepAliveCount = -1;
     private int requestCount = 0;
@@ -332,4 +334,24 @@ public abstract class AbstractSender implements DataSender {
         this.udpPort = udpPort;
     }
 
+
+    public int getUdpRxBufSize() {
+        return udpRxBufSize;
+    }
+
+
+    public void setUdpRxBufSize(int udpRxBufSize) {
+        this.udpRxBufSize = udpRxBufSize;
+    }
+
+
+    public int getUdpTxBufSize() {
+        return udpTxBufSize;
+    }
+
+
+    public void setUdpTxBufSize(int udpTxBufSize) {
+        this.udpTxBufSize = udpTxBufSize;
+    }
+
 }
\ No newline at end of file
diff --git a/java/org/apache/catalina/tribes/transport/ReceiverBase.java b/java/org/apache/catalina/tribes/transport/ReceiverBase.java
index 06d4217c3..50c689920 100644
--- a/java/org/apache/catalina/tribes/transport/ReceiverBase.java
+++ b/java/org/apache/catalina/tribes/transport/ReceiverBase.java
@@ -56,6 +56,9 @@ public abstract class ReceiverBase implements ChannelReceiver, ListenCallback, R
     private int securePort = -1;
     private int rxBufSize = 43800;
     private int txBufSize = 25188;
+    private int udpRxBufSize = 43800;
+    private int udpTxBufSize = 25188;
+
     private boolean listen = false;
     private RxTaskPool pool;
     private boolean direct = true;
@@ -520,4 +523,20 @@ public abstract class ReceiverBase implements ChannelReceiver, ListenCallback, R
         this.udpPort = udpPort;
     }
 
+    public int getUdpRxBufSize() {
+        return udpRxBufSize;
+    }
+
+    public void setUdpRxBufSize(int udpRxBufSize) {
+        this.udpRxBufSize = udpRxBufSize;
+    }
+
+    public int getUdpTxBufSize() {
+        return udpTxBufSize;
+    }
+
+    public void setUdpTxBufSize(int udpTxBufSize) {
+        this.udpTxBufSize = udpTxBufSize;
+    }
+
 }
\ No newline at end of file
diff --git a/java/org/apache/catalina/tribes/transport/nio/NioReceiver.java b/java/org/apache/catalina/tribes/transport/nio/NioReceiver.java
index de527633e..0c91f1f69 100644
--- a/java/org/apache/catalina/tribes/transport/nio/NioReceiver.java
+++ b/java/org/apache/catalina/tribes/transport/nio/NioReceiver.java
@@ -250,6 +250,11 @@ public class NioReceiver extends ReceiverBase implements Runnable, ChannelReceiv
         setListen(true);
         if (selector!=null && datagramChannel!=null) {
             ObjectReader oreader = new ObjectReader(MAX_UDP_SIZE); //max size for a datagram packet
+            datagramChannel.socket().setSendBufferSize(getUdpTxBufSize());
+            datagramChannel.socket().setReceiveBufferSize(getUdpRxBufSize());
+            datagramChannel.socket().setReuseAddress(getSoReuseAddress());
+            datagramChannel.socket().setSoTimeout(getTimeout());
+            datagramChannel.socket().setTrafficClass(getSoTrafficClass());
             registerChannel(selector,datagramChannel,SelectionKey.OP_READ,oreader);
         }
 
diff --git a/java/org/apache/catalina/tribes/transport/nio/NioReplicationTask.java b/java/org/apache/catalina/tribes/transport/nio/NioReplicationTask.java
index 6e3707027..8cb0dbee9 100644
--- a/java/org/apache/catalina/tribes/transport/nio/NioReplicationTask.java
+++ b/java/org/apache/catalina/tribes/transport/nio/NioReplicationTask.java
@@ -308,8 +308,12 @@ public class NioReplicationTask extends AbstractRxTask {
             int total = 0;
             if (channel instanceof DatagramChannel) {
                 DatagramChannel dchannel = (DatagramChannel)channel;
-                while ( total < command.length ) {
-                    total += dchannel.send(buf, udpaddr);
+                //were using a shared channel, it's not thread safe
+                //TODO check optimization, one channel per thread
+                synchronized (dchannel) {
+                    while ( total < command.length ) {
+                        total += dchannel.send(buf, udpaddr);
+                    }
                 }
             } else {
                 while ( total < command.length ) {
diff --git a/java/org/apache/catalina/tribes/transport/nio/NioSender.java b/java/org/apache/catalina/tribes/transport/nio/NioSender.java
index b8e0479ef..3db44e41e 100644
--- a/java/org/apache/catalina/tribes/transport/nio/NioSender.java
+++ b/java/org/apache/catalina/tribes/transport/nio/NioSender.java
@@ -149,8 +149,8 @@ public class NioSender extends AbstractSender implements DataSender{
             socketChannel.socket().setSoLinger(getSoLingerOn(),getSoLingerTime());
             socketChannel.socket().setTrafficClass(getSoTrafficClass());
         } else if (dataChannel!=null) {
-            dataChannel.socket().setSendBufferSize(getTxBufSize());
-            dataChannel.socket().setReceiveBufferSize(getRxBufSize());
+            dataChannel.socket().setSendBufferSize(getUdpTxBufSize());
+            dataChannel.socket().setReceiveBufferSize(getUdpRxBufSize());
             dataChannel.socket().setSoTimeout((int)getTimeout());
             dataChannel.socket().setReuseAddress(getSoReuseAddress());
             dataChannel.socket().setTrafficClass(getSoTrafficClass());
diff --git a/test/org/apache/catalina/tribes/test/channel/TestUdpPackages.java b/test/org/apache/catalina/tribes/test/channel/TestUdpPackages.java
index c159d28a0..dc4f39a73 100644
--- a/test/org/apache/catalina/tribes/test/channel/TestUdpPackages.java
+++ b/test/org/apache/catalina/tribes/test/channel/TestUdpPackages.java
@@ -20,6 +20,8 @@ import junit.framework.TestCase;
 import java.io.Serializable;
 import java.util.Random;
 import java.util.Arrays;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
 
 import org.apache.catalina.tribes.Channel;
 import org.apache.catalina.tribes.ChannelListener;
@@ -33,6 +35,7 @@ import org.apache.catalina.tribes.transport.ReplicationTransmitter;
 import org.apache.catalina.tribes.group.interceptors.MessageDispatchInterceptor;
 import org.apache.catalina.tribes.group.interceptors.MessageDispatch15Interceptor;
 import org.apache.catalina.tribes.group.interceptors.ThroughputInterceptor;
+import org.apache.catalina.tribes.io.XByteBuffer;
 
 /**
  */
@@ -81,11 +84,18 @@ public class TestUdpPackages extends TestCase {
         channel1.send(new Member[] {channel2.getLocalMember(false)}, Data.createRandomData(1024),Channel.SEND_OPTIONS_UDP);
         Thread.sleep(500);
         System.err.println("Finished Single package NO_ACK ["+listener1.count+"]");
-        assertEquals("Checking success messages.",1,listener1.count);
+        assertEquals("Checking success messages.",1,listener1.count.get());
     }
 
     
     public void testDataSendNO_ACK() throws Exception {
+        final AtomicInteger counter = new AtomicInteger(0);
+        ReceiverBase rb1 = (ReceiverBase)channel1.getChannelReceiver();
+        ReceiverBase rb2 = (ReceiverBase)channel2.getChannelReceiver();
+        rb1.setUdpRxBufSize(1024*1024*10);
+        rb2.setUdpRxBufSize(1024*1024*10);
+        rb1.setUdpTxBufSize(1024*1024*10);
+        rb2.setUdpTxBufSize(1024*1024*10);
         System.err.println("Starting NO_ACK");
         Thread[] threads = new Thread[threadCount];
         for (int x=0; x<threads.length; x++ ) {
@@ -93,7 +103,11 @@ public class TestUdpPackages extends TestCase {
                 public void run() {
                     try {
                         long start = System.currentTimeMillis();
-                        for (int i = 0; i < msgCount; i++) channel1.send(new Member[] {channel2.getLocalMember(false)}, Data.createRandomData(1024),Channel.SEND_OPTIONS_UDP);
+                        for (int i = 0; i < msgCount; i++) {
+                            int cnt = counter.addAndGet(1);
+                            channel1.send(new Member[] {channel2.getLocalMember(false)}, Data.createRandomData(1024,cnt),Channel.SEND_OPTIONS_UDP);
+                            Thread.currentThread().sleep(10);
+                        }
                         System.out.println("Thread["+this.getName()+"] sent "+msgCount+" messages in "+(System.currentTimeMillis()-start)+" ms.");
                     }catch ( Exception x ) {
                         x.printStackTrace();
@@ -108,9 +122,19 @@ public class TestUdpPackages extends TestCase {
         for (int x=0; x<threads.length; x++ ) { threads[x].join();}
         //sleep for 50 sec, let the other messages in
         long start = System.currentTimeMillis();
-        while ( (System.currentTimeMillis()-start)<25000 && msgCount*threadCount!=listener1.count) Thread.sleep(500);
+        while ( (System.currentTimeMillis()-start)<25000 && msgCount*threadCount!=listener1.count.get()) Thread.sleep(500);
         System.err.println("Finished NO_ACK ["+listener1.count+"]");
-        assertEquals("Checking success messages.",msgCount*threadCount,listener1.count);
+        System.out.println("Sent "+counter.get()+ " messages. Received "+listener1.count+" Highest msg received:"+listener1.maxIdx);
+        System.out.print("Missing messages:");
+        printMissingMsgs(listener1.nrs,counter.get());
+        assertEquals("Checking success messages.",msgCount*threadCount,listener1.count.get());
+    }
+    
+    public static void printMissingMsgs(int[] msgs, int maxIdx) {
+        for (int i=0; i<maxIdx && i<msgs.length; i++) {
+            if (msgs[i]==0) System.out.print(i+", ");
+        }
+        System.out.println();
     }
 
     public void testDataSendASYNCM() throws Exception {
@@ -136,18 +160,18 @@ public class TestUdpPackages extends TestCase {
             for (int x=0; x<threads.length; x++ ) { threads[x].join();}
             //sleep for 50 sec, let the other messages in
             long start = System.currentTimeMillis();
-            while ( (System.currentTimeMillis()-start)<25000 && msgCount*threadCount!=listener1.count) Thread.sleep(500);
+            while ( (System.currentTimeMillis()-start)<25000 && msgCount*threadCount!=listener1.count.get()) Thread.sleep(500);
             System.err.println("Finished ASYNC MULTI THREAD ["+listener1.count+"]");
-            assertEquals("Checking success messages.",msgCount*threadCount,listener1.count);
+            assertEquals("Checking success messages.",msgCount*threadCount,listener1.count.get());
     }
     public void testDataSendASYNC() throws Exception {
         System.err.println("Starting ASYNC");
         for (int i=0; i<msgCount; i++) channel1.send(new Member[] {channel2.getLocalMember(false)},Data.createRandomData(1024),GroupChannel.SEND_OPTIONS_ASYNCHRONOUS|Channel.SEND_OPTIONS_UDP);
         //sleep for 50 sec, let the other messages in
         long start = System.currentTimeMillis();
-        while ( (System.currentTimeMillis()-start)<5000 && msgCount!=listener1.count) Thread.sleep(500);
+        while ( (System.currentTimeMillis()-start)<5000 && msgCount!=listener1.count.get()) Thread.sleep(500);
         System.err.println("Finished ASYNC");
-        assertEquals("Checking success messages.",msgCount,listener1.count);
+        assertEquals("Checking success messages.",msgCount,listener1.count.get());
     }
 
     public void testDataSendACK() throws Exception {
@@ -155,7 +179,7 @@ public class TestUdpPackages extends TestCase {
         for (int i=0; i<msgCount; i++) channel1.send(new Member[] {channel2.getLocalMember(false)},Data.createRandomData(1024),GroupChannel.SEND_OPTIONS_USE_ACK|Channel.SEND_OPTIONS_UDP);
         Thread.sleep(250);
         System.err.println("Finished ACK");
-        assertEquals("Checking success messages.",msgCount,listener1.count);
+        assertEquals("Checking success messages.",msgCount,listener1.count.get());
     }
 
     public void testDataSendSYNCACK() throws Exception {
@@ -163,24 +187,38 @@ public class TestUdpPackages extends TestCase {
         for (int i=0; i<msgCount; i++) channel1.send(new Member[] {channel2.getLocalMember(false)},Data.createRandomData(1024),GroupChannel.SEND_OPTIONS_SYNCHRONIZED_ACK|GroupChannel.SEND_OPTIONS_USE_ACK|Channel.SEND_OPTIONS_UDP);
         Thread.sleep(250);
         System.err.println("Finished SYNC_ACK");
-        assertEquals("Checking success messages.",msgCount,listener1.count);
+        assertEquals("Checking success messages.",msgCount,listener1.count.get());
     }
 
     public static class Listener implements ChannelListener {
-        long count = 0;
+        AtomicLong count = new AtomicLong(0);
+        int maxIdx = -1;
+        int[] nrs = new int[1000000];
+        public Listener() {
+            Arrays.fill(nrs, 0);
+        }
         public boolean accept(Serializable s, Member m) {
             return (s instanceof Data);
         }
 
         public void messageReceived(Serializable s, Member m) {
-            Data d = (Data)s;
-            if ( !Data.verify(d) ) {
-                System.err.println("ERROR");
-            } else {
-                count++;
-                if ((count %1000) ==0 ) {
-                    System.err.println("SUCCESS:"+count);
+            try {
+                Data d = (Data)s;
+                if ( !Data.verify(d) ) {
+                    System.err.println("ERROR - Unable to verify data package");
+                } else {
+                    long c = count.addAndGet(1);
+                    if ((c%1000) ==0 ) {
+                        System.err.println("SUCCESS:"+c);
+                    }
+                    int nr = d.getNumber();
+                    if (nr>=0 && nr<nrs.length) {
+                        maxIdx = Math.max(maxIdx, nr);
+                        nrs[nr] = 1;
+                    }
                 }
+            }catch (Exception x ) {
+                x.printStackTrace();
             }
         }
     }
@@ -189,25 +227,41 @@ public class TestUdpPackages extends TestCase {
         public int length;
         public byte[] data;
         public byte key;
+        public boolean hasNr = false;
         public static Random r = new Random(System.currentTimeMillis());
         public static Data createRandomData() {
             return createRandomData(ChannelReceiver.MAX_UDP_SIZE);
         }
         public static Data createRandomData(int size) {
+            return createRandomData(size,-1);
+        }
+        
+        public static Data createRandomData(int size, int number) {
             int i = r.nextInt();
             i = ( i % 127 );
             int length = Math.abs(r.nextInt() % size);
+            if (length<100) length += 100;
             Data d = new Data();
             d.length = length;
             d.key = (byte)i;
             d.data = new byte[length];
             Arrays.fill(d.data,d.key);
+            if (number>0 && d.data.length>=4) {
+                //populate number
+                d.hasNr = true;
+                XByteBuffer.toBytes(number,d.data, 0);
+            }
             return d;
         }
+        
+        public int getNumber() {
+            if (!hasNr) return -1;
+            return XByteBuffer.toInt(this.data, 0);
+        }
 
         public static boolean verify(Data d) {
             boolean result = (d.length == d.data.length);
-            for ( int i=0; result && (i<d.data.length); i++ ) result = result && d.data[i] == d.key;
+            for ( int i=(d.hasNr?4:0); result && (i<d.data.length); i++ ) result = result && d.data[i] == d.key;
             return result;
         }
     }
diff --git a/webapps/docs/config/cluster-receiver.xml b/webapps/docs/config/cluster-receiver.xml
index e272910a5..007b2dfb6 100644
--- a/webapps/docs/config/cluster-receiver.xml
+++ b/webapps/docs/config/cluster-receiver.xml
@@ -117,6 +117,14 @@
     <attribute name="txBufSize" required="false">
       The sending buffer size on the receiving sockets. Value is in bytes, the default value is <code>25188</code> bytes.
     </attribute>
+    <attribute name="udpRxBufSize" required="false">
+        The receive buffer size on the datagram socket.
+        Default value is <code>25188</code> bytes.
+    </attribute>
+    <attribute name="udpTxBufSize" required="false">
+       The send buffer size on the datagram socket.
+       Default value is <code>43800</code> bytes.
+    </attribute>
     <attribute name="soKeepAlive" required="false">
       Boolean value for the socket SO_KEEPALIVE option. Possible values are <code>true</code> or <code>false</code>.
     </attribute>
diff --git a/webapps/docs/config/cluster-sender.xml b/webapps/docs/config/cluster-sender.xml
index 5485b946e..fe82c98c9 100644
--- a/webapps/docs/config/cluster-sender.xml
+++ b/webapps/docs/config/cluster-sender.xml
@@ -90,6 +90,14 @@
        The send buffer size on the socket.
        Default value is <code>43800</code> bytes.
       </attribute>
+      <attribute name="udpRxBufSize" required="false">
+        The receive buffer size on the datagram socket.
+        Default value is <code>25188</code> bytes.
+      </attribute>
+      <attribute name="udpTxBufSize" required="false">
+       The send buffer size on the datagram socket.
+       Default value is <code>43800</code> bytes.
+      </attribute>
       <attribute name="direct" required="false">
        Possible values are <code>true</code> or <code>false</code>. 
        Set to true if you want the receiver to use direct bytebuffers when reading data