I want to generate grad cams from images in tensorflow

Question

As I wrote in my title I have trained model having many sub-models. I tried to write grad-cam generation code but it gave me a bunch of errors. Have a look on this code.

import cv2
import numpy as np
from tensorflow.keras.models import Model
import tensorflow as tf
import matplotlib.pyplot as plt

def generate_gradcam(model, features, layer_name="global_vision_encoder"):
    # Pass the features through the model to create the necessary nodes
    model_outputs = model([features["image"], features["impression"], features["finding"]], training=False)
    text_embeddings, image_embeddings = model_outputs

# Get the layer output
layer = model.get_layer(layer_name)

# Create a model that outputs the activations of the specified layer
gradcam_model = Model(inputs=model.inputs, outputs=layer.output)

# Compute the gradients of the predicted class with respect to the layer output
with tf.GradientTape() as tape:
    layer_outputs = gradcam_model([features["image"], features["impression"], features["finding"]], training=False)
    if isinstance(layer_outputs, list):
        layer_outputs = layer_outputs[0]
    if len(layer_outputs.shape) == 4:
        layer_outputs = tf.reduce_mean(layer_outputs, axis=[1, 2])
    predicted_class_idx = tf.argmax(image_embeddings)

grads = tape.gradient(image_embeddings[0, predicted_class_idx], layer_outputs)

# Compute the Grad-CAM heatmap
heatmap = tf.reduce_mean(grads, axis=-1)
heatmap = np.maximum(heatmap, 0) / np.max(heatmap)
heatmap = heatmap.reshape((8, 8))
heatmap = cv2.resize(heatmap, (features["image"].shape[1], features["image"].shape[2]))
heatmap = np.uint8(255 * heatmap)
heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)

# Superimpose the heatmap on the original image
superimposed_img = heatmap * 0.4 + features["image"].numpy()[0]
superimposed_img = np.clip(superimposed_img, 0, 255).astype(np.uint8)

return superimposed_img

# Load and preprocess the chest X-ray image
image_path = r"C:\Users\zahid\Desktop\3.png"
image = cv2.imread(image_path)
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
image = cv2.resize(image, (299, 299))
image = image.astype(np.float32) / 255.0
image = np.expand_dims(image, axis=0)  # Add batch dimension

# Create the input features dictionary
features = {
    "image": tf.convert_to_tensor(image, dtype=tf.float32),
    "impression": tf.convert_to_tensor([""], dtype=tf.string),
    "finding": tf.convert_to_tensor([""], dtype=tf.string)
}

# Generate Grad-CAM highlighting the locations
layer_name = "global_vision_encoder"  # Change this to the desired layer name
gradcam_image = generate_gradcam(clip_model, features, layer_name)

# Display the Grad-CAM image
plt.imshow(gradcam_image)
plt.axis('off')
plt.show()

When I run this code it says

    TypeError                                 Traceback (most recent call last)
Cell In[30], line 60
     58 # Generate Grad-CAM highlighting the locations
     59 layer_name = "global_vision_encoder"  # Change this to the desired layer name
---> 60 gradcam_image = generate_gradcam(clip_model, features, layer_name)
     62 # Display the Grad-CAM image
     63 plt.imshow(gradcam_image)

Cell In[30], line 9, in generate_gradcam(model, features, layer_name)
      7 def generate_gradcam(model, features, layer_name="global_vision_encoder"):
      8     # Pass the features through the model to create the necessary nodes
----> 9     model_outputs = model([features["image"], features["impression"], features["finding"]], training=False)
     10     text_embeddings, image_embeddings = model_outputs
     12     # Get the layer output

File ~\AppData\Local\anaconda3\envs\tu\lib\site-packages\keras\utils\traceback_utils.py:67, in filter_traceback.<locals>.error_handler(*args, **kwargs)
     65 except Exception as e:  # pylint: disable=broad-except
     66   filtered_tb = _process_traceback_frames(e.__traceback__)
---> 67   raise e.with_traceback(filtered_tb) from None
     68 finally:
     69   del filtered_tb

Cell In[17], line 33, in ClipModel.call(self, features, training)
     31 def call(self, features, training=False):
     32     # Get the embeddings for the impressions.
---> 33     impression_embeddings = self.impression_encoder(features["impression"], training=training)
     34     # Get the embeddings for the findings.
     35     finding_embeddings = self.finding_encoder(features["finding"], training=training)

TypeError: Exception encountered when calling layer "clip_model" (type ClipModel).

list indices must be integers or slices, not str

Call arguments received:
  • features=['tf.Tensor(shape=(1, 299, 299, 3), dtype=float32)', 'tf.Tensor(shape=(1,), dtype=string)', 'tf.Tensor(shape=(1,), dtype=string)']
  • training=False

Similarly when I make changes in this code like this.

import cv2
import numpy as np
from tensorflow.keras.models import Model
import tensorflow as tf
import matplotlib.pyplot as plt

def generate_gradcam(model, features, layer_name="global_vision_encoder"):
    # Pass the features through the model to create the necessary nodes
    model_outputs = model({
        "image": features["image"],
        "impression": features["impression"],
        "finding": features["finding"]
    }, training=False)
    text_embeddings, image_embeddings = model_outputs

    # Get the layer output
    layer = model.get_layer(layer_name)

    # Create a model that outputs the activations of the specified layer
    gradcam_model = Model(inputs=model.inputs, outputs=layer.output)

    # Compute the gradients of the predicted class with respect to the layer output
    with tf.GradientTape() as tape:
        layer_outputs = gradcam_model({
            "image": features["image"],
            "impression": features["impression"],
            "finding": features["finding"]
        }, training=False)
        if isinstance(layer_outputs, list):
            layer_outputs = layer_outputs[0]
        if len(layer_outputs.shape) == 4:
            layer_outputs = tf.reduce_mean(layer_outputs, axis=[1, 2])
        predicted_class_idx = tf.argmax(image_embeddings)

    grads = tape.gradient(image_embeddings[0, predicted_class_idx], layer_outputs)

    # Compute the Grad-CAM heatmap
    heatmap = tf.reduce_mean(grads, axis=-1)
    heatmap = np.maximum(heatmap, 0) / np.max(heatmap)
    heatmap = heatmap.reshape((8, 8))
    heatmap = cv2.resize(heatmap, (features["image"].shape[1], features["image"].shape[2]))
    heatmap = np.uint8(255 * heatmap)
    heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)

    # Superimpose the heatmap on the original image
    superimposed_img = heatmap * 0.4 + features["image"].numpy()[0]
    superimposed_img = np.clip(superimposed_img, 0, 255).astype(np.uint8)

    return superimposed_img

# Load and preprocess the chest X-ray image
image_path = r"C:\Users\zahid\Desktop\3.png"
image = cv2.imread(image_path)
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
image = cv2.resize(image, (299, 299))
image = image.astype(np.float32) / 255.0
image = np.expand_dims(image, axis=0)  # Add batch dimension

# Create the input features dictionary
features = {
    "image": tf.convert_to_tensor(image, dtype=tf.float32),
    "impression": tf.convert_to_tensor([""], dtype=tf.string),
    "finding": tf.convert_to_tensor([""], dtype=tf.string)
}

# Generate Grad-CAM highlighting the locations
layer_name = "global_vision_encoder"  # Change this to the desired layer name
gradcam_image = generate_gradcam(clip_model, features, layer_name)

# Display the Grad-CAM image
plt.imshow(gradcam_image)
plt.axis('off')
plt.show()

Then it says

ValueError                                Traceback (most recent call last)
Cell In[31], line 68
     66 # Generate Grad-CAM highlighting the locations
     67 layer_name = "global_vision_encoder"  # Change this to the desired layer name
---> 68 gradcam_image = generate_gradcam(clip_model, features, layer_name)
     70 # Display the Grad-CAM image
     71 plt.imshow(gradcam_image)

Cell In[31], line 20, in generate_gradcam(model, features, layer_name)
     17 layer = model.get_layer(layer_name)
     19 # Create a model that outputs the activations of the specified layer
---> 20 gradcam_model = Model(inputs=model.inputs, outputs=layer.output)
     22 # Compute the gradients of the predicted class with respect to the layer output
     23 with tf.GradientTape() as tape:

File ~\AppData\Local\anaconda3\envs\tu\lib\site-packages\tensorflow\python\training\tracking\base.py:629, in no_automatic_dependency_tracking.<locals>._method_wrapper(self, *args, **kwargs)
    627 self._self_setattr_tracking = False  # pylint: disable=protected-access
    628 try:
--> 629   result = method(self, *args, **kwargs)
    630 finally:
    631   self._self_setattr_tracking = previous_value  # pylint: disable=protected-access

File ~\AppData\Local\anaconda3\envs\tu\lib\site-packages\keras\engine\functional.py:146, in Functional.__init__(self, inputs, outputs, name, trainable, **kwargs)
    143   if not all([functional_utils.is_input_keras_tensor(t)
    144               for t in tf.nest.flatten(inputs)]):
    145     inputs, outputs = functional_utils.clone_graph_nodes(inputs, outputs)
--> 146 self._init_graph_network(inputs, outputs)

File ~\AppData\Local\anaconda3\envs\tu\lib\site-packages\tensorflow\python\training\tracking\base.py:629, in no_automatic_dependency_tracking.<locals>._method_wrapper(self, *args, **kwargs)
    627 self._self_setattr_tracking = False  # pylint: disable=protected-access
    628 try:
--> 629   result = method(self, *args, **kwargs)
    630 finally:
    631   self._self_setattr_tracking = previous_value  # pylint: disable=protected-access

File ~\AppData\Local\anaconda3\envs\tu\lib\site-packages\keras\engine\functional.py:229, in Functional._init_graph_network(self, inputs, outputs)
    226   self._input_coordinates.append((layer, node_index, tensor_index))
    228 # Keep track of the network's nodes and layers.
--> 229 nodes, nodes_by_depth, layers, _ = _map_graph_network(
    230     self.inputs, self.outputs)
    231 self._network_nodes = nodes
    232 self._nodes_by_depth = nodes_by_depth

File ~\AppData\Local\anaconda3\envs\tu\lib\site-packages\keras\engine\functional.py:1036, in _map_graph_network(inputs, outputs)
   1034 for x in tf.nest.flatten(node.keras_inputs):
   1035   if id(x) not in computable_tensors:
-> 1036     raise ValueError(
   1037         f'Graph disconnected: cannot obtain value for tensor {x} '
   1038         f'at layer "{layer.name}". The following previous layers '
   1039         f'were accessed without issue: {layers_with_complete_input}')
   1040 for x in tf.nest.flatten(node.outputs):
   1041   computable_tensors.add(id(x))

ValueError: Graph disconnected: cannot obtain value for tensor KerasTensor(type_spec=TensorSpec(shape=(None, 299, 299, 3), dtype=tf.float32, name='global_image_input'), name='global_image_input', description="created by layer 'global_image_input'") at layer "tf.math.truediv". The following previous layers were accessed without issue: []

I don't know what is wrong, if you need further information about my code, plz let me know in comments. Thank You