I started to Deep learning with Deep Java Library (DJL) and Spring boot. When I train my dataset I got this error
ai.djl.engine.EngineException: MXNet engine call failed: MXNetError: Check failed: assign(&dattr, vec.at(i)): Incompatible attr in node at 1-th input: expected int32, got float32
my full code his her github My implementation of RandomAccessDataset lool like this code online that is my code
package com.anthill.OptimumBet.basic;
import ai.djl.ndarray.NDArray;
import ai.djl.ndarray.NDList;
import ai.djl.ndarray.NDManager;
import ai.djl.training.dataset.RandomAccessDataset;
import ai.djl.training.dataset.Record;
import ai.djl.util.Progress;
import com.anthill.OptimumBet.util.Utils;
import java.io.IOException;
import java.io.Reader;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.List;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
/**
*
* @author truth
*/
public class CSVDataset extends RandomAccessDataset {
private final List<CSVRecord> csvRecords;
private CSVDataset(Builder builder) {
super(builder);
this.csvRecords = builder.dataset;
}
@Override
public Record get(NDManager manager, long index) {
CSVRecord record = csvRecords.get(Math.toIntExact(index));
NDArray datum = manager.create(
encode(record.get(Utils.QUESTION_MALICIOUS_URL)));
NDArray label = manager.create(
Float.parseFloat(record.get(Utils.ANSWER_MALICIOUS_URL)));
return new Record(new NDList(datum), new NDList(label));
}
@Override
protected long availableSize() {
return this.csvRecords.size();
}
private int[] encode(String url) {
url = url.toLowerCase();
int[] encoding = new int[26];
for (char ch : url.toCharArray()) {
int index = ch - 'a';
if (index < 26 && index >= 0) {
encoding[ch - 'a']++;
}
}
return encoding;
}
@Override
public void prepare(Progress prgrs) {}
public static Builder builder(String csvFilePath) {
return new Builder(csvFilePath);
}
public static final class Builder extends BaseBuilder<Builder> {
List<CSVRecord> dataset;
private final String csvFilePath;
private Usage usage;
Builder(String csvFilePath) {
this.csvFilePath = csvFilePath;
this.usage = Usage.TRAIN;
}
@Override
protected Builder self() {
return this;
}
public Builder optUsage(Usage usage) {
this.usage = usage;
return this;
}
public CSVDataset build() throws IOException {
try ( Reader reader = Files.newBufferedReader(Paths.get(csvFilePath));
CSVParser csvParser = new CSVParser(
reader,
CSVFormat.DEFAULT.builder()
.setHeader(Utils.QUESTION_MALICIOUS_URL, Utils.ANSWER_MALICIOUS_URL)
.setSkipHeaderRecord(true)
.setIgnoreHeaderCase(true)
.setTrim(true)
.build())) {
List<CSVRecord> csvRecords = csvParser.getRecords();
int index = (int) (csvRecords.size() * 0.8);
switch (usage) {
case TRAIN: {
dataset = csvRecords.subList(0, index);
break;
}
case TEST: {
dataset = csvRecords.subList(index, csvRecords.size());
break;
}
default:{
dataset = csvRecords;
break;
}
}
}
return new CSVDataset(this);
}
}
}
and that is my training
public TrainingResult trainCSVDataset() throws IOException, TranslateException {
try ( Model model = Model.newInstance(Utils.MODEL_NAME_MALICIOUS_URL)) {
model.setBlock(
new Mlp(
Utils.INPUT_MALICIOUS_URL,
Utils.OUTPUT_MALICIOUS_URL,
new int[]{128, 64}));
//get training and validation dataset
RandomAccessDataset trainingSet = this.getCSVDataSet(
Dataset.Usage.TRAIN,
Utils.BATCH_SIZE,
Utils.LIMIT);
RandomAccessDataset validateSet = this.getCSVDataSet(
Dataset.Usage.TEST,
Utils.BATCH_SIZE,
Utils.LIMIT);
//set training configuration
try ( Trainer trainer = model.newTrainer(
this.setupTrainingConfig(Utils.MODEL_NAME_MALICIOUS_URL,
this.getGpus()))) {
trainer.setMetrics(new Metrics());
trainer.initialize(new Shape(1, Utils.INPUT_MALICIOUS_URL));
System.out.println("trainingDataset size :"+ trainingSet.size());
EasyTrain.fit(trainer, Utils.EPOCH, trainingSet, validateSet);
Path modelDire = Paths.get(Utils.MODEL_DIRE);
Files.createDirectories(modelDire);
model.setProperty("Epoch", String.valueOf(Utils.EPOCH));
model.save(modelDire, Utils.MODEL_NAME_MALICIOUS_URL);
return trainer.getTrainingResult();
}
}
}
private DefaultTrainingConfig setupTrainingConfig(
String outputDire,
int maxGpus) {
SaveModelTrainingListener listener =
new SaveModelTrainingListener(outputDire);
listener.setSaveModelCallback(
trainer -> {
TrainingResult result = trainer.getTrainingResult();
Model model = trainer.getModel();
model.setProperty(
"Accuracy",
String.format(
"%.5f",
result.getTrainEvaluation("Accuracy")));
model.setProperty(
"Loss",
String.format("%.5f", result.getValidateLoss()));
});
return new DefaultTrainingConfig(Loss.softmaxCrossEntropyLoss())
.addEvaluator(new Accuracy())
.optDevices(Engine.getInstance().getDevices(maxGpus))
.addTrainingListeners(
TrainingListener.Defaults.logging(outputDire))
.addTrainingListeners(listener);
}
private RandomAccessDataset getDataSet(
Dataset.Usage usage,
int batchSize,
int limit) throws IOException {
Mnist mnist = Mnist.builder()
.optUsage(usage)
.setSampling(batchSize, true)
.optLimit(limit)
.build();
mnist.prepare(new ProgressBar());
return mnist;
}
private RandomAccessDataset getCSVDataSet(Dataset.Usage usage, int batchSize, int limit) throws IOException, TranslateException {
RandomAccessDataset csvDataset = CSVDataset.builder(Utils.MALICIOUS_URL_DATA_IN_PATH_SYSTEM)
.optUsage(usage)
.setSampling(batchSize, true)
.optLimit(limit)
.build();
// csvDataset.prepare(new ProgressBar());
return csvDataset;
}
private int getGpus() {
return Engine.getInstance().getGpuCount();
}
please someone can help me to fix my error?
that is the full message error
Sun May 01 07:47:06 WAT 2022
There was an unexpected error (type=Internal Server Error, status=500).
MXNet engine call failed: MXNetError: Check failed: assign(&dattr, vec.at(i)): Incompatible attr in node at 1-th input: expected int32, got float32 Stack trace: File "../src/io/../operator/elemwise_op_common.h", line 135
ai.djl.engine.EngineException: MXNet engine call failed: MXNetError: Check failed: assign(&dattr, vec.at(i)): Incompatible attr in node at 1-th input: expected int32, got float32
Stack trace:
File "../src/io/../operator/elemwise_op_common.h", line 135
at ai.djl.mxnet.jna.JnaUtils.checkCall(JnaUtils.java:1930)
at ai.djl.mxnet.jna.JnaUtils.imperativeInvoke(JnaUtils.java:519)
at ai.djl.mxnet.jna.FunctionInfo.invoke(FunctionInfo.java:72)
at ai.djl.mxnet.engine.MxNDManager.invoke(MxNDManager.java:294)
at ai.djl.mxnet.engine.MxNDArrayEx.linear(MxNDArrayEx.java:597)
at ai.djl.nn.core.Linear.linear(Linear.java:182)
at ai.djl.nn.core.Linear.forwardInternal(Linear.java:87)
at ai.djl.nn.AbstractBaseBlock.forwardInternal(AbstractBaseBlock.java:125)
at ai.djl.nn.AbstractBaseBlock.forward(AbstractBaseBlock.java:90)
at ai.djl.nn.SequentialBlock.forwardInternal(SequentialBlock.java:147)
at ai.djl.nn.AbstractBaseBlock.forward(AbstractBaseBlock.java:90)
at ai.djl.training.Trainer.forward(Trainer.java:173)
at ai.djl.training.EasyTrain.trainSplit(EasyTrain.java:120)
at ai.djl.training.EasyTrain.trainBatch(EasyTrain.java:108)
at ai.djl.training.EasyTrain.fit(EasyTrain.java:57)
at com.anthill.OptimumBet.service.TrainModelService.trainCSVDataset(TrainModelService.java:108)
at com.anthill.OptimumBet.controller.OptimumBetController.trainCSVDataSet(OptimumBetController.java:48)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:566)
at org.springframework.web.method.support.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:205)
at org.springframework.web.method.support.InvocableHandlerMethod.invokeForRequest(InvocableHandlerMethod.java:150)
at org.springframework.web.servlet.mvc.method.annotation.ServletInvocableHandlerMethod.invokeAndHandle(ServletInvocableHandlerMethod.java:117)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.invokeHandlerMethod(RequestMappingHandlerAdapter.java:895)
at org.springframework.web.servlet.mvc.method.annotation.RequestMappingHandlerAdapter.handleInternal(RequestMappingHandlerAdapter.java:808)
at org.springframework.web.servlet.mvc.method.AbstractHandlerMethodAdapter.handle(AbstractHandlerMethodAdapter.java:87)
at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:1067)
at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:963)
at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:1006)
at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:898)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:655)
at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:883)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:764)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:227)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)
at org.apache.tomcat.websocket.server.WsFilter.doFilter(WsFilter.java:53)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)
at org.springframework.web.filter.RequestContextFilter.doFilterInternal(RequestContextFilter.java:100)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)
at org.springframework.web.filter.FormContentFilter.doFilterInternal(FormContentFilter.java:93)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)
at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:201)
at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:117)
at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:189)
at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:162)
at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:197)
at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:97)
at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:541)
at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:135)
at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:92)
at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:78)
at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:360)
at org.apache.coyote.http11.Http11Processor.service(Http11Processor.java:399)
at org.apache.coyote.AbstractProcessorLight.process(AbstractProcessorLight.java:65)
at org.apache.coyote.AbstractProtocol$ConnectionHandler.process(AbstractProtocol.java:889)
at org.apache.tomcat.util.net.NioEndpoint$SocketProcessor.doRun(NioEndpoint.java:1743)
at org.apache.tomcat.util.net.SocketProcessorBase.run(SocketProcessorBase.java:49)
at org.apache.tomcat.util.threads.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1191)
at org.apache.tomcat.util.threads.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:659)
at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
at java.base/java.lang.Thread.run(Thread.java:829)
sorry for my english! I'm still learning.
This problem is reported from MXNet engine when invoking the forward function call on the
Linearblock. And data type mismatch is found. But theLinearblock should be rather mature in DJL. There are many test already.Maybe you can set break point to check the dataType of the NDArray, right before the JNA is called, for example, somewhere around
at ai.djl.mxnet.engine.MxNDArrayEx.linear(MxNDArrayEx.java:597), to see if they are as expected. On the other hand, you can take a look atintegration/src/main/java/ai/djl/integration/tests/nn/BlockCoreTest.javato see successful application of Linear block forward call. Use it as a benchmark on the dataType issue.