How to visualize the heat maps

[FreeCoderCW]

Excuse me. Thanks for your work, but I wonder how to visualize the heat maps in your paper. Could you please show the code or explain the method about it.

[exitudio]

Hi,
We use GradCAM to visualize where the model attend to. Since, we didn't clean up the visualization code so we remove it but you still can see our code from .ipynb in history.

Here is the part to generate GradCAM using this GradCAM lib

from pytorch_grad_cam import GradCAM
from pytorch_grad_cam.utils.model_targets import ClassifierOutputTarget
def grad_cam_graph(target, plot=True, reshape_transform=lambda x:x):
    input_tensor = data_target[target].unsqueeze(dim=0)
    input_tensor = rearrange(input_tensor, 'b f j d -> b d f j')
    cam = GradCAM(model=model, target_layers=target_layers, use_cuda=True, reshape_transform=reshape_transform)
    targets = [ClassifierOutputTarget(124)]

    grayscale_cam = cam(input_tensor=input_tensor, targets=targets)
    grayscale_cam = grayscale_cam[0, :]

    if plot:
        plot_activation_graph(data_target[target]*320, grayscale_cam)
    return grayscale_cam

def plot_gradcam(data):
    y_axis = ["nose","left_eye","right_eye","left_ear","right_ear","left_shoulder","right_shoulder","left_elbow","right_elbow","left_wrist","right_wrist","left_hip","right_hip","left_knee","right_knee","left_ankle","right_ankle"]
    fig, ax = plt.subplots(1,1)
    plt.yticks(fontsize=6)
    ax.set_yticks(np.arange(len(y_axis)))
    ax.set_yticklabels(y_axis)
    ax.imshow(data)

checkpoint = torch.load('/home/epinyoan/git/GaitSelfFormer/v2_all/save/unify/ablation_study/2022-10-04-12-40-45_11_nolocal_32_64_128_256_lr6e-3/checkpoint/last.pth')
# checkpoint = torch.load('/home/epinyoan/git/GaitSelfFormer/v2_all/save/unify/ablation_study/2022-10-04-13-50-45_12_globallocal_32_64_128_256_lr6e-3/checkpoint/last.pth')
# checkpoint = torch.load('/home/epinyoan/git/GaitSelfFormer/v2_all/save/unify/ablation_study/2022-10-11-15-30-34_25_no_l2norm_lr6e-3/checkpoint/last.pth')
model = SpatialTransformerTemporalConv(
            num_frame=60, in_chans=2, spatial_embed_dim=32, out_dim=128, num_joints=17, kernel_frame=31)
target_layers = [model.conv4]

model.load_state_dict(checkpoint, strict=True)
model = nn.Sequential(Rearrange('b d f j -> b f j d'), model)

# (191, (75, 2, 2, 18), (98, 0, 2, 18)),
grayscale_cam = grad_cam_graph((89, 2, 2, 36), plot=False)
# plt.imshow(grayscale_cam.T)
plot_gradcam(grayscale_cam.T)