How to Emulate a remote JVM in JUnit using Spring, Ant, H2 and ActiveMQ

How to Emulate a remote JVM in JUnit using Spring, Ant, H2 and ActiveMQ

recently i needed to test a scenario where one JVM was communicating to another using both a shared database and JMS.  i also needed to have each JVM be "invisible" to each other and make sure that the couldn't 'accidentally' share things across and application context.

so, to do this, i needed to kick start a separate JVM from inside a JUnit test, and communicate via JMS and a database... fun.

here's what i needed;

- shared database (standalone server)
- shared JMS (standalone server)
- JVM A - application context with unique configuration
- JVM B - application context with unique configuration

so, here's the solution i put together with some google/stackoverflow help and the following;
- Spring
- JUnit 4
- Ant
- H2
- ActiveMQ

here's the infrastructure utilities class.  it's responsible for setting up 'shared' infrastructure such as H2 and ActiveMQ.  it also provides hooks for binding to JNDI in a standalone context to help with sharing the resources in an 'agnostic' kind of way.

package de.incompleteco.spring.batch.ha;

import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;

import javax.sql.DataSource;

import org.apache.activemq.broker.BrokerService;
import org.apache.activemq.command.ActiveMQQueue;
import org.apache.activemq.spring.ActiveMQConnectionFactory;
import org.h2.jdbcx.JdbcConnectionPool;
import org.h2.jdbcx.JdbcDataSource;
import org.h2.tools.Server;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.mock.jndi.SimpleNamingContextBuilder;

public class InfrastructureUtils {

 private static final Logger logger = LoggerFactory.getLogger(InfrastructureUtils.class);
 
 private static Server server;
 
 private static BrokerService broker;
 
 public static void startH2() throws Exception {
  if (server == null) {
   server = Server.createTcpServer("-tcpAllowOthers");
   server.start();
   logger.info(server.getStatus());
  }//end if
 } 
 
 public static void stopH2() throws Exception {
  if (server != null) {
   server.stop();
  }//end if
 } 
 
 public static void startAMQ() throws Exception {
  if (broker == null) {
   broker = new BrokerService();
   broker.addConnector("tcp://localhost:61616");
   broker.setUseJmx(false);
   broker.setUseShutdownHook(true);
   broker.start();
   broker.deleteAllMessages();//clean up
  }//end if
 }
 
 public static void stopAMQ() throws Exception {
  if (broker != null) {
   broker.stop();
  }//end if
 } 
 
 public static void bindLocalAMQ(String connectionFactoryName,String... queueNames) {
  //get jndi
  SimpleNamingContextBuilder builder = SimpleNamingContextBuilder.getCurrentContextBuilder();
  //get the connection
  ActiveMQConnectionFactory connectionFactory = new ActiveMQConnectionFactory();
  connectionFactory.setBrokerURL("tcp://localhost:61616");
  //bind
  builder.bind("jms/" + connectionFactoryName,connectionFactory);
  //setup the queues
  for (String name : queueNames) {
   builder.bind("jms/" + name,new ActiveMQQueue(name));   
  }//end for
 } 
 
 public static DataSource bindLocalH2(String dataSourceName) throws Exception {
  JdbcDataSource dataSource = new JdbcDataSource();
  dataSource.setURL("jdbc:h2:tcp://localhost/~/test");
  //build a pool and bind
  JdbcConnectionPool pool = JdbcConnectionPool.create(dataSource);
  SimpleNamingContextBuilder builder = SimpleNamingContextBuilder.getCurrentContextBuilder();
  builder.bind("jdbc/" + dataSourceName, pool);
  //return it in case there's other uses
  return dataSource;
 } 
 
 public static String[] convertSqlFile(String location) throws Exception {
  List sqlStatements = new ArrayList();
  //load up the file
  BufferedReader reader = new BufferedReader(new InputStreamReader(TestSimpleBatchHAService.class.getResourceAsStream(location)));
  String line;
  StringBuilder statement = new StringBuilder();
  while ((line = reader.readLine()) != null) {
   if (line.contains(";") && !line.contains("--")) {
    statement.append(line);
    sqlStatements.add(statement.toString().replace(';',' '));
    statement = new StringBuilder();//reset the string
   } else if (line.contains("--")) {
    //ignore
   } else {
    statement.append(line);
   }//end if
  }//end while
  reader.close();
  //return
  return sqlStatements.toArray(new String[sqlStatements.size()]);
 } 
}

here's the RemoteJVMEmulator class.  it's a 'standalone' class (static void main(..)) that bootstraps and application context.  it also uses the infrastructure utilities to bind the resources to the JNDI.

package de.incompleteco.spring.batch.ha;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.mock.jndi.SimpleNamingContextBuilder;

public class RemoteJVMEmulator {

 private static final Logger logger = LoggerFactory.getLogger(RemoteJVMEmulator.class);
 
 //bind shared resources to JNDI
 public static void setupJNDI() throws Exception {
  SimpleNamingContextBuilder.emptyActivatedContextBuilder();
  //bind AMQ
  InfrastructureUtils.bindLocalAMQ("ConnectionFactory","request.queue","reply.queue");
  //bind h2
  InfrastructureUtils.bindLocalH2("DataSource");
 }
 
 public static void main(String[] args) throws Exception {
  //setup jndi
  setupJNDI();
  //start the app context
  ApplicationContext context = new ClassPathXmlApplicationContext(args);
  //print a statement saying "it's up"
  logger.info(RemoteJVMEmulator.class.getSimpleName() + " is up " + context.getStartupDate());
 }

}

and here's the test runner.  this uses the infrastructure utilities to startup the shared services and bind them locally.  it then uses and inner class to build and run and Ant task that will kick off the RemoteJVMEmulator class.  a couple of things to note about it though;
- the Ant task is started in a separate thread - mainly because it blocks
- the Ant task is started first to act as the 'remote client'/'listener'
- the 'remote client'/'listener' needs to be setup such that you don't need to be in the same JVM to access any information from it.  (treat it as a 'remote' server deployed and running without any interaction services - you could connect to it via JMX or expose other services, that's up to you)

package de.incompleteco.spring.batch.ha;

import static org.junit.Assert.assertFalse;

import java.io.PrintStream;

import javax.sql.DataSource;

import org.apache.tools.ant.DefaultLogger;
import org.apache.tools.ant.DemuxOutputStream;
import org.apache.tools.ant.Project;
import org.apache.tools.ant.taskdefs.Java;
import org.apache.tools.ant.types.Path;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.springframework.batch.core.Job;
import org.springframework.batch.core.JobExecution;
import org.springframework.batch.core.JobParameters;
import org.springframework.batch.core.JobParametersBuilder;
import org.springframework.batch.core.explore.JobExplorer;
import org.springframework.batch.core.launch.JobLauncher;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.mock.jndi.SimpleNamingContextBuilder;

//don't run this test in CI
@Ignore
public class TestSimpleBatchHAService {

 
 @BeforeClass
 public static void beforeClass() throws Exception {
  //start a builder
  SimpleNamingContextBuilder.emptyActivatedContextBuilder();
  //setup the services (h2 and amq)
  InfrastructureUtils.startH2();
  //setup the database
  DataSource dataSource = InfrastructureUtils.bindLocalH2("DataSource");
  setupH2Data(dataSource);
  //start amq
  InfrastructureUtils.startAMQ();
  //bind
  InfrastructureUtils.bindLocalAMQ("ConnectionFactory","request.queue","reply.queue");
 }
 
 @AfterClass
 public static void afterClass() throws Exception {
  //shutdown the services
  InfrastructureUtils.stopH2();
  InfrastructureUtils.stopAMQ();
 }
 
 private static void setupH2Data(DataSource dataSource) throws Exception {
  //execute the statements
  JdbcTemplate template = new JdbcTemplate(dataSource);
  //drop statements
  String[] statements = InfrastructureUtils.convertSqlFile("/org/springframework/batch/core/schema-drop-h2.sql");
  try {
   for (String statement : statements) {
    template.execute(statement);
   }//end for
  }
  catch (Exception e) { }
  statements = InfrastructureUtils.convertSqlFile("/META-INF/sql/schema-ext-drop-h2.sql");
  try {
   for (String statement : statements) {
    template.execute(statement);
   }//end for
  }
  catch (Exception e) { }  
  //create statements
  statements = InfrastructureUtils.convertSqlFile("/org/springframework/batch/core/schema-h2.sql");
  for (String statement : statements) {
   template.execute(statement);
  }//end for  
  //create statements
  statements = InfrastructureUtils.convertSqlFile("/META-INF/sql/schema-ext-h2.sql");
  for (String statement : statements) {
   template.execute(statement);
  }//end for    
 }
 
 
 @Test
 public void testExecute() throws Exception {
  //start a thread to run the remote
  new Thread(new RemoteJVMRunner()).start();
  //now start the 'server'
  ApplicationContext context = new ClassPathXmlApplicationContext("classpath:/META-INF/spring/server-context.xml");
  //now that it's started, run the job
  Job job = context.getBean(Job.class);
  JobParameters parameters = new JobParametersBuilder().addLong("runtime",System.currentTimeMillis()).toJobParameters();
  JobLauncher launcher = context.getBean("remoteJobLauncher", JobLauncher.class);
  JobExecution execution = launcher.run(job,parameters);
  JobExplorer explorer = context.getBean(JobExplorer.class);
  //monitor
  while (explorer.getJobExecution(execution.getId()).isRunning()) {
   Thread.sleep(500);
  }//end while
  //reload the execution
  execution = explorer.getJobExecution(execution.getId());
  //check
  assertFalse(execution.getStatus().isUnsuccessful());
 }

 
 class RemoteJVMRunner implements Runnable {
  
  @Override
  public void run() {
   Project project = new Project();
   project.setName("remote-jvm");
   project.init();
   //setup the logger
   DefaultLogger logger = new DefaultLogger();
   project.addBuildListener(logger);
   logger.setOutputPrintStream(System.out);
   logger.setErrorPrintStream(System.err);
   logger.setMessageOutputLevel(Project.MSG_INFO);
   System.setOut(new PrintStream(new DemuxOutputStream(project,false)));
   System.setErr(new PrintStream(new DemuxOutputStream(project,true)));
   //start the project
   project.fireBuildStarted();
   
   Java java = new Java();
   java.setProject(project);;
   java.setTaskName("run-remote-jvm");
   java.setFork(true);
   java.setFailonerror(true);
   //set the classname
   java.setClassname(RemoteJVMEmulator.class.getName());
   java.setClasspath(new Path(project,System.getProperty("java.class.path")));
   //create arguments
   java.createArg().setValue("classpath:/META-INF/spring/client-context.xml");
   //init
   java.init();
   //execute
   java.executeJava();
  }
 }
}

Spring Batch High Availability

Spring Batch High Availability

here's a quick project i recently threw together to demonstrate how to setup a high availability cluster for processing Spring Batch.  HA doesn't mean what you typically think, but in a batch environment, it allows for a configurable period of time in which should a job 'stop running' (that is, seem to be running but the underlying JVM has failed), the job will be 'failed' and restarted immediately on another cluster. the design is as follows;

Master/Heartbeat Consumer/Launcher

Overview

- JVM that starts Jobs on itself or other nodes
- Heartbeat Consumer - runs a TCP socket server that listens for 'heartbeats' from other JVM's
- Is responsible for failing over a job to another server should the job fail where it is

Key Components

- MessageChannelJobLauncher - uses Spring Integration to serialize the Job's name and it's parameters and send it out as a start request
- BatchHAService - service to be periodically invoked and check for that executions registered are still running/stable - responsible for failing and restarting jobs that become 'ghosts'
- BatchHeartbeatConsumerService - responsible for consuming heartbeats and registering job execution ids

Client/Heartbeat Producer

Overview

- JVM that actually runs a Job
- produces Heartbeats - a list of registered/in-flight job execution ids
- uses a RemoteJobRegistry to persist job names in a shared batch database

Key Components

- JobExecutionRegisterListener - wraps any jobs and registers the job execution id in the current JVM
- JobExecutionRegisterListenerPostProcessor - responsible for implementing the Listener on all Jobs
- RemoteJobRegistry - wraps the 'JVM local' JobRegistry and persists in a shared database, any Job names on the local JVM
- BatchHeartbeatClientService - responsible for accepting jobExecution registers and publishing

The infrastructure pieces are;
- Job "start" requests are sent via JMS
- there is a shared Job Database (batch database) that includes a new table, Job_Entity for persisting job names
- TCP server/client implemented in Spring Integration to publish the Job Execution Ids as part of the heart beat

the code will be up on github soon and i'll flesh it out properly, but it really is magic what Spring Integration can do.

Spring Integration, ActiveMQ Dead Letter Queue and Testing

Spring Integration, ActiveMQ Dead Letter Queue and Testing

recently i was working on a design whereby the application would consume messages from a JMS queue, but may encounter a transient exception (e.g. the database connection may not be available temporarily).  in Spring Integration i could use a retry pattern on the endpoint, but as the application is in a cluster, another JVM may have better luck getting to the database.  so, to use the cluster design, if a transient exception should occur, the exception would be thrown up, the transaction would roll back and the message would go back on the queue, awaiting consumption again.  this would allow another node in the cluster an opportunity to get the message and process it.

ok, so this is all very nice, but how do we confirm that this actually works?  we're using ActiveMQ in the test environment (embedded) and wanted to see the message actually got rolled back.  the default behavior of ActiveMQ, (since version 5.4?) is the following;

• no delay in retries
• 6 attempts
• if all attempt fail, put it on the Dead Letter Queue

so, here's a couple of snippets that use this to test.

first the integration setup;

***note: a DefaultMessageListenerContainer is manually configured to support the transactions, setting the transactionManager directly on the channel adapter alone doesn't work (in 2.2.4.RELEASE)

now our test resources;

our testing bean;

package de.incompleteco.spring.integration.service;

public class FailureService {

 public void process(Object payload) {
  throw new RuntimeException("simulated failure");
 }
 
}

and our junit test;

package de.incompleteco.spring.integration;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import javax.jms.ConnectionFactory;
import javax.jms.Message;
import javax.jms.Queue;
import javax.jms.TextMessage;

import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.jms.core.JmsTemplate;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration({"classpath*:/META-INF/spring/*-context.xml"})
@ActiveProfiles("junit")
public class DeadLetterQueueIntegrationTest {

 private static final String PAYLOAD = "hello world";
 
 @Autowired
 private ConnectionFactory connectionFactory;
 
 @Autowired
 @Qualifier("request.queue")
 private Queue requestQueue;
 
 @Autowired
 @Qualifier("dead.letter.queue")
 private Queue dlQueue;
 
 private JmsTemplate jmsTemplate;
 
 @Before
 public void before() throws Exception {
  jmsTemplate = new JmsTemplate(connectionFactory);
  jmsTemplate.setDefaultDestination(dlQueue);
  jmsTemplate.setReceiveTimeout(2 * 1000);//about 2 seconds
 }
 
 @Test
 public void test() throws Exception {
  //send a message
  jmsTemplate.convertAndSend(requestQueue, PAYLOAD);
  //expect it to fail and, by default, be put back on the dead letter queue
  Message failedMessage = jmsTemplate.receive();
  //now check that it has the same content
  assertTrue(failedMessage instanceof TextMessage);
  assertEquals(PAYLOAD,((TextMessage) failedMessage).getText());
 }
 
}

Spring Integration, Spring JMS, ActiveMQ and testing Topics

Spring Integration, Spring JMS, ActiveMQ and testing Topics

so everyone knows how to test a JMS Queue in JUnit, ActiveMQ and - to make life easy - Spring JmsTemplate.  but, testing a Topic is a slightly different matter.

first, a topic subscriber has to be 'connected' before you can publish the message.  that's a bit of a challenge to setup.  but, with a little Spring Integration to the rescue we can get a subscriber sorted and sne a message via the JmsTemplate.

another catch comes from ActiveMQ.  to test queues, you can setup the ActiveMQConnectionFactory and you're done.  however, with Topics, you'll need the PooledConnectionFactory as well.

first, a couple dependencies for our pom

   
    org.apache.activemq
    activemq-core
    5.6.0
    test
   
   
    org.apache.activemq
    activemq-pool
    5.6.0
    test
   

then a little Spring Integration to listen to the Topic

(i prefer a queue channel, just to make it easier to get the messages off in testing)
so the next part would be the test class

package de.incompleteco.spring.integration.test;

import static org.junit.Assert.assertNotNull;

import javax.jms.ConnectionFactory;
import javax.jms.JMSException;
import javax.jms.Session;
import javax.jms.Topic;

import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.integration.Message;
import org.springframework.integration.core.PollableChannel;
import org.springframework.jms.core.JmsTemplate;
import org.springframework.jms.core.MessageCreator;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;

@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration({"classpath:/META-INF/spring/integration-context.xml"})
public class TopicIntegrationTest {

 @Autowired
 private PollableChannel receiveChannel;
 
 @Autowired
 private ConnectionFactory connectionFactory;
 
 @Autowired
 private Topic topic;
 
 private JmsTemplate jmsTemplate;
 
 @Test
 public void test() throws Exception {
  //setup the jmsTemplate and send
  jmsTemplate = new JmsTemplate(connectionFactory);
  jmsTemplate.setDefaultDestination(topic);
  jmsTemplate.setPubSubDomain(true);//it's a topic
  //send
  jmsTemplate.send(new MessageCreator() {

   public javax.jms.Message createMessage(Session session) throws JMSException {
    return session.createTextMessage("hello world");
   } });
  //now grab it off the queue
  Message message = receiveChannel.receive(1000);//let's put a timeout
  //test
  assertNotNull(message);
 }
 
}

and here's the specific apache configuration to support Topics

Spring Batch Partition Example

Spring Batch Partition Example

up on github (https://github.com/incomplete-code/spring-batch-partition) is a project where i've put a small sample of Spring Batch Partition with both a taskExecutor pattern (commented out) and a local JMS version (setup).