I have a requirement to process thousands of Payments..So I used VtdXml instead of StaxEventItemReader in Spring Batch, I have created Custom Item Reader for the same. In order to read huge xml with Multi Threading, I have created partition with 10 Threads. I am splitting huge xml file into 10 files and assigning to each Thread in Partition. Once I read the xml and I will convert into list of Objects and send to Writer. After received in Writer, I will customize the List of Objects and Merge into the final List. Whenever Am returning list of Objects read called again and it is never ending. How do I pass the list of Objects to the Writer and merge into the final List?
public class VtdWholeItemReader<T> implements ResourceAwareItemReaderItemStream<T> {
private Resource resource;
private boolean noInput;
private boolean strict = true;
private InputStream inputStream;
private int index = 0;
@Override
public void open(ExecutionContext executionContext) {
Assert.notNull(resource, "The Resource must not be null.");
noInput = true;
if (!resource.exists()) {
if (strict) {
throw new IllegalStateException("Input resource must exist (reader is in 'strict' mode)");
}
log.warn("Input resource does not exist " + resource.getDescription());
return;
}
if (!resource.isReadable()) {
if (strict) {
throw new IllegalStateException("Input resource must be readable (reader is in 'strict' mode)");
}
log.warn("Input resource is not readable " + resource.getDescription());
return;
}
noInput = false;
}
@Override
public void update(ExecutionContext executionContext) {
}
@Override
public void close() {
try {
if (inputStream != null) {
inputStream.close();
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
inputStream = null;
}
}
@Override
public void setResource(Resource resource) {
this.resource = resource;
}
@Override
public T read()
throws java.lang.Exception, UnexpectedInputException, ParseException, NonTransientResourceException {
if (noInput) {
return null;
}
List<Payment> paymentList = new ArrayList<Payment>();
try {
VTDGen vg = new VTDGen();
VTDGen vgHen = new VTDGen();
boolean headercheck = true;
if (vg.parseFile("src/main/resources/input/partitioner/" + resource.getFilename(), false)) {
VTDNav vn = vg.getNav();
AutoPilot ap = new AutoPilot(vn);
ap.selectXPath("/root/Payment");
// flb contains all the offset and length of the segments to be skipped
FastLongBuffer flb = new FastLongBuffer(4);
int i;
byte[] xml = vn.getXML().getBytes();
while ((i = ap.evalXPath()) != -1) {
flb.append(vn.getElementFragment());
}
int size = flb.size();
log.info("Payment Size {}", size);
if (size != 0) {
for (int k = 0; k < size; k++) {
String message = new String(xml, flb.lower32At(k), flb.upper32At(k), StandardCharsets.UTF_8);
ObjectMapper objectMapper = new ObjectMapper();
objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
Payment payment = objectMapper
.readValue(message, Payment.class);
paymentList.add(pcPayment);
index = pcPaymentList.size() + 1;
}
}
log.info("Payment List:: {}", paymentList.size());
log.info("Index::{}", index);
return index > paymentList .size() ? null : (T) paymentList;
}
} catch (Exception e) {
e.printStackTrace();
}
return null;
}
}
SpringBatch ConfigClass
private final Logger logger = LoggerFactory.getLogger(SpringBatchConfig.class);
@Autowired
private JobBuilderFactory jobBuilderFactory;
@Autowired
private StepBuilderFactory stepBuilderFactory;
@Autowired
ResourcePatternResolver resoursePatternResolver;
@Bean
public Job job() {
return jobBuilderFactory.get("job").start(readpayment()).build();
}
@Bean
public JobLauncher jobLauncher() throws Exception {
SimpleJobLauncher jobLauncher = new SimpleJobLauncher();
jobLauncher.setJobRepository(jobRepository());
jobLauncher.afterPropertiesSet();
return jobLauncher;
}
@Bean
public JobRepository jobRepository() throws Exception {
MapJobRepositoryFactoryBean factory = new MapJobRepositoryFactoryBean();
factory.setTransactionManager(new ResourcelessTransactionManager());
return (JobRepository) factory.getObject();
}
@Bean
protected Step readpayment() {
return stepBuilderFactory.get("readpayment").partitioner("paymentStep", partitioner(null))
.step(paymentStep()).taskExecutor(taskExecutor()).build();
}
@Bean
protected Step paymentStep() {
return stepBuilderFactory.get("paymentStep")
.<Payment,Payment>chunk(10)
.reader(xmlFileItemReader(null))
.writer(writer()).build();
}
@Bean
public TaskExecutor taskExecutor() {
ThreadPoolTaskExecutor taskExecutor = new ThreadPoolTaskExecutor();
taskExecutor.setMaxPoolSize(10);
taskExecutor.setCorePoolSize(10);
taskExecutor.setQueueCapacity(10);
taskExecutor.afterPropertiesSet();
return taskExecutor;
}
@Bean
@StepScope
ItemReader<Payment> xmlFileItemReader(@Value("#{stepExecutionContext[fileName]}") String filename) {
VtdWholeItemReader<Payment> xmlFileReader = new VtdWholeItemReader<>();
xmlFileReader.setResource(new ClassPathResource("input/partitioner/" + filename));
return xmlFileReader;
}
@Bean
@StepScope
public CustomMultiResourcePartitioner partitioner(@Value("#{jobParameters['fileName']}") String fileName) {
logger.info("fileName {}", fileName);
CustomMultiResourcePartitioner partitioner = new CustomMultiResourcePartitioner();
Resource[] resources;
try {
resources = resoursePatternResolver.getResources("file:src/main/resources/input/partitioner/*.xml");
} catch (IOException e) {
throw new RuntimeException("I/O problems when resolving the input file pattern.", e);
}
partitioner.setResources(resources);
return partitioner;
}
@Bean
public ItemWriter<Payment> writer() {
return new PaymentItemWriter();
}
PaymentItemWriter
@Override
public void write(List<? extends List<Payment>> items) throws Exception {
log.info("Items {}", items.size());
}
May be try making ItemWriter bean as step scope ["@StepScope"] in Spring batch configuration class