The above image shows the structure of my data.
from sklearn.model_selection import train_test_split
from datasets import Features, ClassLabel, Value, Dataset, DatasetDict
df_train, df_tmp = train_test_split(
movie_df,stratify=movie_df["label"], test_size=0.2)
df_val, df_test = train_test_split(
df_tmp,stratify=df_tmp["label"], test_size=0.5)
ds_features = Features({"text": Value("string"), "label": ClassLabel(names=labels)})
dataset = DatasetDict({
"train": Dataset.from_pandas(df_train.reset_index(drop=True),features=ds_features),
"valid": Dataset.from_pandas(df_val.reset_index(drop=True),features=ds_features),
"test": Dataset.from_pandas(df_test.reset_index(drop=True),features=ds_features)})
dataset
this code gave me a value error as shown:
I was expecting something similar to this but not with the same values:
DatasetDict({
train: Dataset({
features: ['text', 'label'],
num_rows: 13267
})
valid: Dataset({
features: ['text', 'label'],
num_rows: 1658
})
test: Dataset({
features: ['text', 'label'],
num_rows: 1659
})
})
Can anyone tell me what I am doing wrong?