Encountering Fitting Errors when Implementing TensorFlow: Seeking Assistance

Question

I am trying to rerun the code provided here (https://www.analyticsvidhya.com/blog/2021/05/tuning-the-hyperparameters-and-layers-of-neural-network-deep-learning/). I have made the necessary changes to continue using Keras2. However, I keep encountering this error.

ValueError: All the 5 fits failed. It is very likely that your model is misconfigured. You can try to debug the error by setting error_score='raise'.

Below are more details about the failures:

5 fits failed with the following error: Traceback (most recent call last): File "/opt/anaconda3/envs/deeplearning_tf/lib/python3.9/site-packages/sklearn/model_selection/_validation.py", line 895, in _fit_and_score estimator.fit(X_train, y_train, **fit_params) File "/opt/anaconda3/envs/deeplearning_tf/lib/python3.9/site-packages/scikeras/wrappers.py", line 1491, in fit super().fit(X=X, y=y, sample_weight=sample_weight, **kwargs) File "/opt/anaconda3/envs/deeplearning_tf/lib/python3.9/site-packages/scikeras/wrappers.py", line 760, in fit self._fit( File "/opt/anaconda3/envs/deeplearning_tf/lib/python3.9/site-packages/scikeras/wrappers.py", line 915, in _fit X, y = self._initialize(X, y) File "/opt/anaconda3/envs/deeplearning_tf/lib/python3.9/site-packages/scikeras/wrappers.py", line 852, in initialize self.model = self._build_keras_model() File "/opt/anaconda3/envs/deeplearning_tf/lib/python3.9/site-packages/scikeras/wrappers.py", line 429, in _build_keras_model model = final_build_fn(**build_params) File "/var/folders/1j/wm50c4515n50js7rns8cfj5r0000gq/T/ipykernel_42818/2602284124.py", line 63, in nn_cl_fun nn.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) File "/opt/anaconda3/envs/deeplearning_tf/lib/python3.9/site-packages/keras/src/utils/traceback_utils.py", line 122, in error_handler raise e.with_traceback(filtered_tb) from None File "/opt/anaconda3/envs/deeplearning_tf/lib/python3.9/site-packages/keras/src/optimizers/init.py", line 97, in get raise ValueError(f"Could not interpret optimizer identifier: {identifier}") ValueError: Could not interpret optimizer identifier: <tf_keras.src.optimizers.legacy.adagrad.Adagrad object at 0x30621dd60>

import numpy as np
import pandas as pd
import tf_keras as tf_keras
import os
os.environ["TF_USE_LEGACY_KERAS"] = 'True'
from sklearn.model_selection import train_test_split, cross_val_score, StratifiedKFold
from keras.models import Sequential
from keras.layers import Dense, BatchNormalization, Dropout, LeakyReLU
from tf_keras.optimizers.legacy import Adam, SGD, RMSprop, Adagrad, Ftrl
from keras.callbacks import EarlyStopping
from scikeras.wrappers import KerasClassifier
from math import floor
from sklearn.metrics import make_scorer, accuracy_score
from bayes_opt import BayesianOptimization

LeakyReLU = LeakyReLU(alpha=0.1)
score_acc = make_scorer(accuracy_score)

trainSet = pd.read_csv('/Users/asam/Desktop/D_Drive/ET_Prediction/layers_tune/train.csv')
train = trainSet.drop(columns=['Name', 'Ticket', 'Cabin']).dropna(axis=0)
train = pd.get_dummies(train)

X_train, X_val, y_train, y_val = train_test_split(train.drop(columns=['PassengerId','Survived'], axis=0),
                                                  train['Survived'],
                                                  test_size=0.2, random_state=111,
                                                  stratify=train['Survived'])

def nn_cl_bo2(neurons, activation, optimizer, learning_rate, batch_size, epochs,
              layers1, layers2, normalization, dropout, dropout_rate):
    optimizerL = ['SGD', 'Adam', 'RMSprop', 'Adadelta', 'Adagrad', 'Adamax', 'Nadam', 'Ftrl','SGD']
    optimizerD= {'Adam':Adam(learning_rate), 'SGD':SGD(learning_rate),
                 'RMSprop':RMSprop(learning_rate), 
                 'Adagrad':Adagrad(learning_rate), 'Ftrl':Ftrl(learning_rate)}
    activationL = ['relu', 'sigmoid', 'softplus', 'softsign', 'tanh', 'selu',
                   'elu', 'exponential', LeakyReLU,'relu']
    neurons = round(neurons)
    activation = activationL[round(activation)]
    optimizer = optimizerD[optimizerL[round(optimizer)]]
    batch_size = round(batch_size)
    epochs = round(epochs)
    layers1 = round(layers1)
    layers2 = round(layers2)
    
    def nn_cl_fun():
        nn = Sequential()
        nn.add(Dense(neurons, input_dim=10, activation=activation))
        if normalization > 0.5:
            nn.add(BatchNormalization())
        for i in range(layers1):
            nn.add(Dense(neurons, activation=activation))
        if dropout > 0.5:
            nn.add(Dropout(dropout_rate, seed=123))
        for i in range(layers2):
            nn.add(Dense(neurons, activation=activation))
        nn.add(Dense(1, activation='sigmoid'))
        nn.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])
        return nn
    
    es = EarlyStopping(monitor='accuracy', mode='max', verbose=0, patience=20)
    nn = KerasClassifier(model=nn_cl_fun, epochs=epochs, batch_size=batch_size, verbose=0)
    kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=123)
    score = cross_val_score(nn, X_train, y_train, scoring=score_acc, cv=kfold, fit_params={'callbacks':[es]}).mean()
    return score

params_nn2 ={
    'neurons': (10, 100),
    'activation':(0, 9),
    'optimizer':(0,4),
    'learning_rate':(0.01, 1),
    'batch_size':(200, 1000),
    'epochs':(20, 100),
    'layers1':(1,3),
    'layers2':(1,3),
    'normalization':(0,1),
    'dropout':(0,1),
    'dropout_rate':(0,0.3)
}
nn_bo = BayesianOptimization(nn_cl_bo2, params_nn2, random_state=111)
nn_bo.maximize(init_points=25, n_iter=4)