I am trying to build a Keras model for my reinforcement learning algorithm, but I am encountering a problem related to an error about an invalid output shape. Here is the code for my build_model function:
def build_model(observation_space, action_space):
# Input for possible_position_map matrix
possible_position_map_input = Input(shape=observation_space['possible_position_map'].shape, name='possible_position_map_input')
# Reshape input data to fit Conv2D format
possible_position_map_reshaped = Reshape((100, 1, 1))(possible_position_map_input)
# Convolutional layers for processing possible_position_map matrix
possible_position_map_conv = Conv2D(filters=16, kernel_size=(3, 1), activation='relu')(possible_position_map_reshaped)
possible_position_map_flatten = Flatten()(possible_position_map_conv)
# Input for height_map matrix
height_map_input = Input(shape=observation_space['height_map'].shape, name='height_map_input')
# Reshape input data to fit Conv2D format
height_map_input_reshaped = Reshape((100, 1, 1))(height_map_input)
# Convolutional layers for processing height_map matrix
height_map_conv = Conv2D(filters=16, kernel_size=(3, 1), activation='relu')(height_map_input_reshaped)
height_map_flatten = Flatten()(height_map_conv)
# Input for currentBox vector
current_box_input = Input(shape=observation_space['currentBox'].shape, name='current_box_input')
current_box_flatten = Flatten()(current_box_input)
# Merge all branches
merged = Concatenate()([possible_position_map_flatten, height_map_flatten, current_box_flatten])
# Dense layers for action decisions
dense1 = Dense(128, activation='relu')(merged)
dense2 = Dense(64, activation='relu')(dense1)
# Output layer for discrete actions
output = Dense(action_space.n, activation='softmax')(dense2)
model = tf.keras.Model(inputs=[possible_position_map_input, height_map_input, current_box_input], outputs=output)
return model
Invocation:
env = ENV.env.BinEnv(False, False)
model = ENV.Model.build_model(env.observation_space, env.action_space)
model.summary()
policy = BoltzmannQPolicy()
memory = SequentialMemory(limit=50000, window_length=1)
dqn = DQNAgent(model=model, memory=memory, policy=policy,
nb_actions=env.action_space.n, nb_steps_warmup=10, target_model_update=1e-2)
dqn.compile(Adam(lr=1e-3), metrics=['mae'])
However, I'm getting the following error:
File "C:\Users\konra\.conda\envs\magisterka\Lib\site-packages\rl\agents\dqn.py", line 107, in __init__
raise ValueError(f'Model output "{model.output}" has invalid shape. DQN expects a model that has one dimension for each action, in this case {self.nb_actions}.')
ValueError: Model output "Tensor("dense_2/Softmax:0", shape=(?, 100), dtype=float32)" has invalid shape. DQN expects a model that has one dimension for each action, in this case 100.
Can someone help me identify the cause of this error and how to fix it?
Model: "model"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
possible_position_map_inpu [(None, 100)] 0 []
t (InputLayer)
height_map_input (InputLay [(None, 100)] 0 []
er)
reshape (Reshape) (None, 100, 1, 1) 0 ['possible_position_map_input[
0][0]']
reshape_1 (Reshape) (None, 100, 1, 1) 0 ['height_map_input[0][0]']
conv2d (Conv2D) (None, 98, 1, 16) 64 ['reshape[0][0]']
conv2d_1 (Conv2D) (None, 98, 1, 16) 64 ['reshape_1[0][0]']
current_box_input (InputLa [(None, 3)] 0 []
yer)
flatten (Flatten) (None, 1568) 0 ['conv2d[0][0]']
flatten_1 (Flatten) (None, 1568) 0 ['conv2d_1[0][0]']
flatten_2 (Flatten) (None, 3) 0 ['current_box_input[0][0]']
concatenate (Concatenate) (None, 3139) 0 ['flatten[0][0]',
'flatten_1[0][0]',
'flatten_2[0][0]']
dense (Dense) (None, 128) 401920 ['concatenate[0][0]']
dense_1 (Dense) (None, 64) 8256 ['dense[0][0]']
dense_2 (Dense) (None, 100) 6500 ['dense_1[0][0]']
==================================================================================================
I've tried verifying that the output shape of my model matches the number of actions in my environment (env.action_space.n), and it seems to be correct. I've also double-checked the input shapes and ensured that all layers are connected properly.
entire trace back:
Model: "model"
__________________________________________________________________________________________________
Layer (type) Output Shape Param # Connected to
==================================================================================================
x (InputLayer) [(None, 100)] 0 []
fx (InputLayer) [(None, 3)] 0 []
theta (InputLayer) [(None, 100)] 0 []
dense (Dense) (None, 64) 6464 ['x[0][0]']
dense_1 (Dense) (None, 64) 256 ['fx[0][0]']
dense_2 (Dense) (None, 64) 6464 ['theta[0][0]']
concatenate (Concatenate) (None, 192) 0 ['dense[0][0]',
'dense_1[0][0]',
'dense_2[0][0]']
dense_3 (Dense) (None, 64) 12352 ['concatenate[0][0]']
dense_4 (Dense) (None, 100) 6500 ['dense_3[0][0]']
==================================================================================================
Total params: 32036 (125.14 KB)
Trainable params: 32036 (125.14 KB)
Non-trainable params: 0 (0.00 Byte)
__________________________________________________________________________________________________
Shape of the model output: (?, 100)
Expected shape for DQN: (None, 100)
Traceback (most recent call last):
File "C:\Users\konra\.conda\envs\magisterka\Lib\site-packages\keyboard\_generic.py", line 22, in invoke_handlers
if handler(event):
^^^^^^^^^^^^^^
File "C:\Users\konra\.conda\envs\magisterka\Lib\site-packages\keyboard\__init__.py", line 474, in <lambda>
return hook(lambda e: e.event_type == KEY_UP or callback(e), suppress=suppress)
^^^^^^^^^^^
File "E:\studia\ISA\magisterka\Magisterka\main.py", line 328, in obsluz_zdarzenie
keras()
File "E:\studia\ISA\magisterka\Magisterka\main.py", line 275, in keras
dqn = DQNAgent(model=model, nb_actions=nb_actions, memory=memory, nb_steps_warmup=2000,
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\konra\.conda\envs\magisterka\Lib\site-packages\rl\agents\dqn.py", line 107, in __init__
raise ValueError(f'Model output "{model.output}" has invalid shape. DQN expects a model that has one dimension for each action, in this case {self.nb_actions}.')
ValueError: Model output "Tensor("dense_4/BiasAdd:0", shape=(?, 100), dtype=float32)" has invalid shape. DQN expects a model that has one dimension for each action, in this case 100.