MultiThread causado lentidão no glassfish até travar

Bom dia!

Eu desenvolvi uma rotina em que através desse @Schedule( second=“0/15” ,minute="",hour="", persistent=true)
é iniciado um pool de threads com um numero maximo de 800, ou seja, inicia 800 threads a cada 15 segundos.

Uma nova sequencia de threads são iniciadas somente quando terminar de processar essas 800. A medida que passa o tempo vai tornando cada vez mais lento o glassfish até que ele trava.

Para que volte tenho que parar o glassfish e reinicia-lo. Alguem tem idéia de como resolver?

o código principal é esse:

package com.suppcomm.schedule;

import java.io.IOException;
import java.sql.Timestamp;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.Iterator;
import java.util.List;

import javax.ejb.EJB;
import javax.ejb.Schedule;
import javax.ejb.Stateless;
import javax.servlet.ServletContext;
import javax.ws.rs.core.Context;

import org.apache.log4j.Level;
import org.apache.log4j.Logger;

import br.com.supportcomm.dialout.util.Dialout;
import br.com.supportcomm.mktcall.constants.StatusOperation;
import br.com.supportcomm.mktcall.constants.StatusProcessing;
import br.com.supportcomm.mktcall.entity.Campanha;
import br.com.supportcomm.mktcall.entity.Dialing;
import br.com.supportcomm.mktcall.entity.Insertion;
import br.com.supportcomm.mktcall.exception.MktCallException;
import br.com.supportcomm.mktcall.log.LogMktCall;
import br.com.supportcomm.mktcall.service.campanha.CampanhaService;
import br.com.supportcomm.mktcall.service.dialing.ConfigDelegate;
import br.com.supportcomm.mktcall.service.dialing.DialingDelegate;
import br.com.supportcomm.mktcall.service.insertion.InsertionService;
import br.com.supportcomm.mktcall.tools.SCTools;

import com.suppcomm.util.JSFUtil;

@Stateless
public class TimerDialOut {
Logger logger = Logger.getLogger(getClass());

static final Logger errorLog = Logger.getLogger("logError");
static final Logger infoLog = Logger.getLogger("logInfo");
private static volatile boolean emExec = false;
@EJB
private DialingDelegate dialingDelegate;

@EJB
private ConfigDelegate configDelegate;

@EJB
private CampanhaService campanhaService;

@EJB 
private InsertionService insertionService;

private static synchronized boolean mudaExec(boolean val) {
    if (val == emExec) {
        return false;
    }
    else {
        emExec = val;
        return true;
    }
}




@Context ServletContext servletContext;


 @Schedule( second="0/15" ,minute="*",hour="*", persistent=true)
 public void doWork() throws MktCallException 
 {
        if (!TimerDialOut.mudaExec(true)) {
            LogMktCall.log(Level.ERROR, "Timer captura  in process!");
            return;
        }

        try {
            JSFUtil util = new JSFUtil();
            long horaAtual = util.getRigthNowInMinutes();
            
            String timeRangeStartHour = configDelegate.getValueByIndentify("startProcessHour");
            String timeRangeEndHour   = configDelegate.getValueByIndentify("endProcessHour");
            
            long startProcessHour = util.getHoursInMinutes(configDelegate.getValueByIndentify("startProcessHour"));
            long endProcessHour   = util.getHoursInMinutes(configDelegate.getValueByIndentify("endProcessHour"));

            if(!( horaAtual >= startProcessHour && horaAtual<=endProcessHour)){
                infoLog.info("Hour out off bound!!");
                return;
            }
            
            infoLog.info("processing....");
            List<Channel> channels = new ArrayList<>();
            String context           = configDelegate.getValueByIndentify("context");
            String exter             = configDelegate.getValueByIndentify("extension");
            
            String ipConnection      = configDelegate.getValueByIndentify("serverIP");
            String loginAsterisk     = configDelegate.getValueByIndentify("login");
            String passAsterisk      = configDelegate.getValueByIndentify("senha");
            String channelPrefix     = configDelegate.getValueByIndentify("channelPrefix");
            String channelSuffix     = configDelegate.getValueByIndentify("channelSuffix");
            String timeout           = configDelegate.getValueByIndentify("timeout");
            String startLuaCondition = configDelegate.getValueByIndentify("startLuaCondition");
            Integer priority         = Integer.parseInt( configDelegate.getValueByIndentify("priority"));
            Integer maxResults       = Integer.parseInt( configDelegate.getValueByIndentify("maxResults"));
            Integer timoutDelayMinutes= Integer.parseInt( configDelegate.getValueByIndentify("timoutDelayMinutes"));
            String laNumber=configDelegate.getValueByIndentify("laNumber");
            
            
            //vamos verificar se tem algo in process com mais de 5 minutos.
            //dialingDelegate.verfyInProcess();
            
            List<Campanha>  campanhas=  campanhaService.getCampanhaElegiveis();
            
            //for (Campanha campanha1:campanhas){
                List<Dialing> dialings = dialingDelegate.getListToProcess(campanhas, maxResults);
                if( !(dialings== null)  &&   !dialings.isEmpty()){
                    infoLog.info("Selecionado --" + dialings.size() + " msisdns");
                    
                    
                    setInProcess(channels, laNumber, dialings);
                    
                    List<Dialout> dialouts = new ArrayList<Dialout>();
                    String [] splintIpConnection=ipConnection.split(";");
                    int contador=1;
                    for(Channel chanel:channels){
                        
                        
                        callAsterisk(context, exter, loginAsterisk,
                                passAsterisk, channelPrefix, channelSuffix,
                                timeout, startLuaCondition, priority,
                                dialouts, splintIpConnection, contador,
                                chanel);
                        
                        if (contador++ >= splintIpConnection.length){
                            contador=1;
                        }
                    }
                    
                    boolean acabou=false;
                    List<Dialing> dialoutRespostas = new ArrayList<>();
                    contador=1;
                    while(!acabou){
                        Iterator<Dialout> dials = dialouts.iterator();
                        
                        //for (Iterator<Dialout> dials = dialouts.iterator(); dials.hasNext(); ){
                        while(dials.hasNext()){
                            Dialout dialout = dials.next();
                            if(dialout.getStatus()!=null){
                                
                                infoLog.info("processado --" + contador + " msisdns");
                                contador = trataRetorno(timeRangeStartHour,
                                             timeRangeEndHour,
                                             timoutDelayMinutes, 
                                             dialouts,
                                             dialoutRespostas, 
                                             dialout,contador);
                                dials.remove();
                                
                            }
                        }
                        if (dialouts.size()==0){
                            dialouts = new ArrayList<Dialout>();
                            dials = null;
                            acabou=true;
                            contador=1;
                        }
                        Thread.sleep(10);
                        
                    }
                    
                    for (Dialing dialing:dialoutRespostas ){
                        dialingDelegate.update(dialing);
                    }
                    dialoutRespostas = new ArrayList<>();
                    
                }     // if size

            //} // end for
            
        }
        
        catch(NoClassDefFoundError e) {
             StringBuilder sb = new StringBuilder();
             for (StackTraceElement element : e.getStackTrace()) {
                    sb.append(element.toString());
                    sb.append("\n");
                }
             infoLog.info(sb.toString());
            }
        catch (Exception e) {
            e.printStackTrace();
             StringBuilder sb = new StringBuilder();
             for (StackTraceElement element : e.getStackTrace()) {
                    sb.append(element.toString());
                    sb.append("\n");
                }

             infoLog.info(sb.toString());
        }
        finally {
            TimerDialOut.mudaExec(false);
            infoLog.info("terminate");
        }
 }




/**
 * Chama o asterisk e coloca em thread
 * @param context
 * @param exter
 * @param loginAsterisk
 * @param passAsterisk
 * @param channelPrefix
 * @param channelSuffix
 * @param timeout
 * @param startLuaCondition
 * @param priority
 * @param dialouts
 * @param splintIpConnection
 * @param contador
 * @param chanel
 * @throws IOException
 * @throws NumberFormatException
 */
private void callAsterisk(String context, String exter,
        String loginAsterisk, String passAsterisk, String channelPrefix,
        String channelSuffix, String timeout, String startLuaCondition,
        Integer priority, List<Dialout> dialouts,
        String[] splintIpConnection, int contador, Channel chanel)
        throws IOException, NumberFormatException {
    List<String> variables = new ArrayList<>();
    variables.add( startLuaCondition+"="+exter );
    
    variables.add("destinationNumber=".concat(chanel.getMsisdn()));            
    variables.add("idCampanha=".concat(chanel.getIdCampanha()));
    Timestamp t= new Timestamp(System.currentTimeMillis());
    
    variables.add("answerdatetime=".concat( t.toString()));
    
    Dialout dialout = new Dialout();
    
    dialout.setCallerId(chanel.getMsisdnOriginator());
    if(!chanel.getMsisdn().contains("192")){
        dialout.setChannel(channelPrefix.concat(chanel.getMsisdn().concat(channelSuffix) ));
        dialout.setMsisdnOriginator(chanel.getMsisdn());
        
    }else{
        dialout.setChannel( channelPrefix.replace("0", "").concat(chanel.getMsisdn()));
    }
    
    dialout.setContext(context);
    dialout.setExten(exter);
    dialout.setIpConnection(splintIpConnection[contador-1]);
    if (contador==splintIpConnection.length){
        contador=1;
    }else{
        contador++;
    }

    dialout.setLoginAsterisk(loginAsterisk);
    dialout.setPassAsterisk(passAsterisk);
    dialout.setPriority(priority);
    dialout.setTimeout(Long.parseLong( timeout));
    dialout.setVariables(variables);
    dialout.setMsisdnOriginator(chanel.getMsisdnOriginator());
    dialout.setObject(chanel);
    dialout.setMsisdnDestination(chanel.getMsisdn());
    Thread threadDoPdf = new Thread(dialout);
    threadDoPdf.start();
    dialouts.add(dialout);
}




/**
 * Seta todas as linhas do Dilialing como status in process '999'
 * @param channels
 * @param laNumber
 * @param dialings
 */
private void setInProcess(List<Channel> channels, String laNumber,
        List<Dialing> dialings) {
    for (Dialing dialing:dialings){
        Channel channel = new Channel();
        channel.setIdDialing(dialing.getDialingId());
        channel.setMsisdn(dialing.getMsisdnDialing());
        channel.setIdCampanha( String.valueOf(dialing.getIdCampanha().getIdCampanha()));
        if(dialing.getIdCampanha().getLaCampanha()==null || dialing.getIdCampanha().getLaCampanha().isEmpty()){
            channel.setMsisdnOriginator(laNumber);
        }
        else{
            channel.setMsisdnOriginator(dialing.getIdCampanha().getLaCampanha());
        }    
        
        channels.add(channel);
        dialingDelegate.setInProcess(dialing.getDialingId());
    }
}




private Integer trataRetorno(String timeRangeStartHour,
        String timeRangeEndHour, Integer timoutDelayMinutes,
        List<Dialout> dialouts, List<Dialing> dialoutRespostas,
        Dialout dialout,Integer contator) throws Exception {
    if(!dialout.getResponseCode().equalsIgnoreCase(String.valueOf( StatusProcessing.SUCCESS.value()))){
        printError(dialout);
    } else{
        saveSuccess(dialout,dialoutRespostas );    
    }
    
    if(!dialout.getResponseCode().equalsIgnoreCase(String.valueOf( StatusProcessing.SUCCESS.value()))){
            saveNotSuccess(timeRangeStartHour,
                           timeRangeEndHour,
                           timoutDelayMinutes, dialout,dialoutRespostas);
    }
    //dialouts.remove(dialout);
    return contator ++;
    
}




private void saveNotSuccess(String timeRangeStartHour,
        String timeRangeEndHour, Integer timoutDelayMinutes, Dialout dialout,List<Dialing> dialoutRespostas)
        throws Exception {
    Channel channel = (Channel) dialout.getObject();
    Dialing dialing = dialingDelegate.load(channel.getIdDialing());
    increaseAttemps(dialing);
    dialing.setResponseMessage( dialout.getResponseMessage());
    dialing.setResponseCode(dialout.getResponseCode());
    dialing.setDatetimeScheduled(SCTools.calculateDelayedTimestamp(timoutDelayMinutes, timeRangeStartHour, timeRangeEndHour));
    //dialingDelegate.update(dialing);
    dialoutRespostas.add(dialing);
}




private void printError(Dialout dialout) {
    infoLog.info("---------------------------------------------------");
    Channel channel = (Channel) dialout.getObject();
    infoLog.info("Campanha-----"+ channel.getIdCampanha() );
    infoLog.info("Chamada-----"+ dialout.getChannel() );
    infoLog.info("msisdn-----"+ dialout.getMsisdnDestination());
    infoLog.info("Response Code -------------------"+dialout.getResponseCode());
    infoLog.info("Response Message -------------------"+dialout.getResponseMessage());
    infoLog.info("Status------------------------------" + dialout.getStatus());
    infoLog.info("Exception Stack trace Messgem-------" + (dialout.getMessage()==null?"Sem mensagem":dialout.getMessage() ));
}




private void saveSuccess(Dialout dialout,List<Dialing> dialoutRespostas) throws Exception {
    infoLog.info("Sucesso msisdn-----"+ dialout.getMsisdnDestination());
    
    Channel channel = (Channel) dialout.getObject();
    infoLog.info("Campanha-----"+ channel.getIdCampanha() );
    Dialing dialing = dialingDelegate.load(channel.getIdDialing());
    dialing.setStatus(StatusOperation.OK.value());
    dialing.setResponseMessage(  dialout.getResponseMessage());
    dialing.setResponseCode( dialout.getResponseCode());
    increaseAttemps(dialing); 
    dialing.setDatetimeLastOperation(new Timestamp(System.currentTimeMillis()));
    
    
    dialoutRespostas.add(dialing);
}




private void increaseAttemps(Dialing dialing) {
    if (!SCTools.isNullOrBlank(dialing.getAttempts()))
    {
        dialing.setAttempts(dialing.getAttempts().intValue() + 1);
    } else {
        dialing.setAttempts(1);
    }
}



 private class Channel{
     private String msisdn;
     private String msisdnOriginator;
     private Long idDialing;
     private String idCampanha;
    public String getMsisdn() {
        return msisdn;
    }
    public void setMsisdn(String msisdn) {
        this.msisdn = msisdn;
    }
    public Long getIdDialing() {
        return idDialing;
    }
    public void setIdDialing(Long idDialing) {
        this.idDialing = idDialing;
    }
    public String getMsisdnOriginator() {
        return msisdnOriginator;
    }
    public void setMsisdnOriginator(String msisdnOriginator) {
        this.msisdnOriginator = msisdnOriginator;
    }
    public String getIdCampanha() {
        return idCampanha;
    }
    public void setIdCampanha(String idCampanha) {
        this.idCampanha = idCampanha;
    }

        
 }


 
    private Campanha getCampanhaFiltro(Campanha campanha, Timestamp dataHoje) throws ParseException {
        List<Insertion> insertionList;
        long campanhaTocada=0; 
        
        List<Campanha> campanhasComSchedules = new ArrayList<>();
        List<Campanha> campanhasSemSchedules = new ArrayList<>();
        
            //somento os ativos
            if (campanha.getStatus() == 1) {
                SimpleDateFormat formato = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");  
                String startTime =  campanha.getStartDatatime().toString().substring(0, 11) + "00:00:00";
                String endTime   =  campanha.getEndDatetime().toString().substring(0, 11) + "23:59:59";
                Timestamp stamp = new Timestamp(System.currentTimeMillis());
                Date hoje = new Date(dataHoje.getTime());
                
                Date startDate = formato.parse(startTime);
                Date endDate   = formato.parse(endTime);

                
                if(endDate.compareTo(hoje)>=0  &&    startDate.compareTo(hoje)<=0)
                 {
                    
                            insertionList = insertionService.getInsertionCampanha(campanha.getIdCampanha());
                            campanha.setInsertions(insertionList);
                            if (!campanha.getInsertions().isEmpty()) {
                                if(campanhaTocada!=campanha.getIdCampanha()){
                                    if (verifyDay(campanha)) {
                                        campanhasComSchedules.add(campanha);
                                    }
                                }
                            }else{
                                campanhasSemSchedules.add(campanha);
                            }
                    
                }
            }


        if (!campanhasComSchedules.isEmpty()){
            campanha=campanhasComSchedules.get(0);
        }else{
            if (!campanhasSemSchedules.isEmpty()){
                campanha=campanhasSemSchedules.get(0);
            }
        }
        
        return campanha;
        
    }
 

    private boolean verifyDay(Campanha campanha) {

        Timestamp dataDeHoje = new Timestamp(System.currentTimeMillis());
        
        boolean dayOK = false;
        Calendar now = Calendar.getInstance();
        now.setTime(dataDeHoje);
        int diaDaSemana =now.get(Calendar.DAY_OF_WEEK);
        if (campanha.getInsertions().isEmpty()){
            dayOK = true;
        }
        
        for (Insertion insertion : campanha.getInsertions()) {

            if (insertion.getDayOfWeek() == diaDaSemana ) {
                
                insertion.getStartTime().toString();
                SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                String startTime;
                String endTime;
                String horaDeHoje; 
                startTime =  dataDeHoje.toString().substring(0, 11) + insertion.getStartTime().toString().substring(11,16)+":00";
                endTime   =  dataDeHoje.toString().substring(0, 11) + insertion.getEndTime().toString().substring(11,16)+":00";
                horaDeHoje = dataDeHoje.toString();
                ;
                try {
                    if (sdf.parse(startTime).equals(sdf.parse(horaDeHoje) ) ||sdf.parse(startTime).before(sdf.parse(horaDeHoje))) {
                        if (sdf.parse(endTime).equals(sdf.parse(horaDeHoje) ) ||sdf.parse(endTime).after(sdf.parse(horaDeHoje))) {
                            dayOK = true;
                        }
                    
                    }
                } catch (ParseException e) {
                    infoLog.warn("Método verifyDay - Parse Exception - message: " + e.getMessage());
                    e.printStackTrace();
                }
            }

        }
        return dayOK;

    }

}