How to update to support kernel 4.1x?

[CharmingYang0]

I prepared a latest version Centos (version: 8.2, kernel: 4.18.0-193.19.1.el8_2.x86_64) VMware VM to experience this tool.

The detail system info as below:
CentOS version: 8.2.2004
kernel version: 4.18.0-193.19.1.el8_2.x86_64
systemtap version: 4.2

function scsi_next_command had been removed from SCSI driver,
so now I want to know which function should be probed to substitute it?

#! /usr/bin/env stap

%{
#include <linux/types.h>
#include <scsi/scsi_cmnd.h>
#include <scsi/scsi_device.h>
#include <linux/timer.h>
#include <linux/blkdev.h>
#include <linux/bio.h>
#include <linux/mm_types.h>
#include <linux/mm.h>
#include <linux/fs.h>
%}

global target_access
global target_scmd
global target_rq
global target_bio
global target_r1bio
global target_block
global fix_write
global temp_failure

global error_type
global access_type
global dev_major
global dev_minor_min
global dev_minor_max
global inode_lba_flag
global inode_lba_val
global timeout_flag
global retry_allowed
global target_minor


probe begin
{
        target_block = -1
        target_minor = -1
        printf("\nBEGIN\n")
}


function set_sense_buf:long (cmd:long, result:long, sensekey:long, asc:long, ascq:long)
%{
        struct scsi_cmnd * scmd = (struct scsi_cmnd *)(long)STAP_ARG_cmd;

        scmd->result = (int)(long)STAP_ARG_result; 
        scmd->sense_buffer[0] = 0x70; /* current, fixed format */
        scmd->sense_buffer[2] = (unsigned char)(long)STAP_ARG_sensekey;
        scmd->sense_buffer[7] = 0x13; /* length */
        scmd->sense_buffer[12] = (unsigned char)(long)STAP_ARG_asc;
        scmd->sense_buffer[13] = (unsigned char)(long)STAP_ARG_ascq;
%}

function get_inode:long (page:long)
%{
    struct page * thispage = (struct page *)(long)STAP_ARG_page;
    unsigned long tempval = (unsigned long)thispage->flags;
	struct address_space *mapping = thispage->mapping;

	if (unlikely(PageSwapCache(thispage)))
		mapping = NULL;
#ifdef CONFIG_SLUB
	else if (unlikely(PageSlab(thispage)))
		mapping = NULL;
#endif
	else if (unlikely((unsigned long)mapping & PAGE_MAPPING_ANON))
		mapping = NULL;

        if((mapping != NULL) && (mapping->host != NULL))
        {
                STAP_RETVALUE = (unsigned long)(mapping->host->i_ino);
        } else {
                STAP_RETVALUE = 0;
        }
%}


probe kernel.function("scsi_decide_disposition@drivers/scsi/scsi_error.c")
{
        scmd_direction = $scmd->sc_data_direction 

        if((((temp_failure == 1) || (error_type == 1)) && (target_scmd == $scmd)) && ((scmd_direction == access_type) || (access_type == 3) || ((scmd_direction == 2) && (access_type== 4))) && ($scmd->request->rq_disk != 0))
        {
                major = $scmd->request->rq_disk->major
                minor = $scmd->request->rq_disk->first_minor
                block = $scmd->request->__sector
                req_len = $scmd->sdb->length

                if(major == dev_major && minor == target_minor && ((block == target_block) || ((block <= target_block) && (target_block < block + (req_len >> 9)))))
                {
                        if((scmd_direction == 2) && (fix_write == 2))
                        { 
                                #fix_write = 0
                        } else 
                        {
                                printf("scsi_decide_disposition : major=%d minor=%d scmd=%d \n",major, minor, $scmd)
                                /* create fake status and sense data */
                                temp_failure++
                                set_sense_buf($scmd, 0x02, 0x03, 0x11, 0x04)
                        }
                }
        }
}

//probe kernel.function("scsi_next_command@drivers/scsi/scsi_lib.c") 
//{
//        if((target_access != 0) && (target_scmd == $cmd))
//        {
//              printf("scsi_next_command : cmd = %d  \n", $cmd)
//              target_access = 0
//              target_scmd = 0
//              target_rq = 0
//              restore_state = 0
//        }
//}

probe kernel.function("scsi_dispatch_cmd@drivers/scsi/scsi_lib.c") 
{
        struct_bio= $cmd->request->bio
        block = $cmd->request->__sector
        req_len = $cmd->sdb->length

        if($cmd->request->rq_disk != 0)
        {
                major = $cmd->request->rq_disk->major
                minor = $cmd->request->rq_disk->first_minor
        }

        if(target_block == -1)
        {
                if(struct_bio != 0)
                {
                        page = $cmd->request->bio->bi_io_vec->bv_page
                        if(page != 0)
                        {
                                inode = get_inode(page)
                        }
                }
        }

        if(((inode_lba_flag ==1)&&(inode == inode_lba_val)) || ((inode_lba_flag ==0 ) && ((block <= inode_lba_val) && (inode_lba_val < block + (req_len >> 9)))) || (block == target_block) || ((block <= target_block) && (target_block < block + (req_len >> 9))))
        {
                printf("\nSCSI_DISPATCH_CMD: command= %d  \n", $cmd->cmnd[0])
                printf("SCSI_DISPATCH_CMD: major= %d minor= %d \n", major, minor)
                printf("SCSI_DISPATCH_CMD: flag(0:LBA, 1:inode)= %d \n", inode_lba_flag)
                printf("SCSI_DISPATCH_CMD: start sector= %d \n", $cmd->request->__sector)
                printf("SCSI_DISPATCH_CMD: req bufflen= %d \n", $cmd->sdb->length)
                printf("SCSI_DISPATCH_CMD: inode= %d \n", inode)
                printf("SCSI_DISPATCH_CMD: scmd = %d \n", $cmd)
                printf("SCSI_DISPATCH_CMD: [7]=%d [8]=%d \n", $cmd->cmnd[7],$cmd->cmnd[8])

                if((target_minor== -1) && (major == dev_major) && ((dev_minor_min & 0xfff0) <= minor) && (minor <= (dev_minor_max & 0xfff0)))
                {
                        tmp_minor = minor
                }

                if((major == dev_major && ((minor == tmp_minor) || (minor == target_minor))) && (fix_write != 2))
                {
                        /* inject errors on the designated device */
                        printf("SCSI_DISPATCH_CMD: cmd-retries = %d entire-retry =%d \n", $cmd->retries, entire_retries)
                        cmd_direction = $cmd->sc_data_direction
                        if((cmd_direction == 1) && (access_type == 4))
                        {
                                fix_write = 2
                                printf("SCSI_DISPATCH_CMD: fix_write =%d \n", fix_write)
                        }

                        if((temp_failure == 0) || (error_type == 1) ||((timeout_flag == 1) && (entire_retries <= retry_allowed)))
                        {
                                if((cmd_direction == access_type) || ((cmd_direction == 2) && (access_type == 4)) || (access_type == 3))
                                {
                                        if(target_minor == -1)
                                        {
                                                target_minor = tmp_minor
                                        }

                                        if(target_block == -1)
                                        {
                                                target_block = block
                                        }

                                        if(target_access == 0)
                                        {
                                                retry_allowed = $cmd->allowed
                                                target_access++
                                                target_scmd = $cmd
                                                target_rq = $cmd->request
                                        }

                                        temp_failure++

                                        if(($cmd->cmnd[0] == 0x28) || ($cmd->cmnd[0] == 0x2a))
                                        {
                                                /* read_10 or write_10 */
                                                $cmd->cmnd[7]=0
                                                $cmd->cmnd[8]=0
                                        }else if(($cmd->cmnd[0] == 0x08) || ($cmd->cmnd[0] == 0x0a))
                                        {
                                                /* read_16 or write_16 */
                                                $cmd->cmnd[10]=0
                                                $cmd->cmnd[11]=0
                                                $cmd->cmnd[12]=0
                                                $cmd->cmnd[13]=0
                                        }else if(($cmd->cmnd[0] == 0x08) || ($cmd->cmnd[0] == 0x0a))
                                        {
                                                /* read_6 or write_6 */
                                                $cmd->cmnd[4]=0
                                        }

                                        if(target_scmd == $cmd)
                                        {
                                                entire_retries++
                                        }

                                        if((target_access_t == 0) && (timeout_flag == 1))
                                        {
                                                target_access_t++
                                                global_scmd = $cmd
                                                restore_state = $cmd->device->host->shost_state
                                                $cmd->device->host->shost_state = 4
                                        }
                                }
                        }

                        printf("\nSCSI_DISPATCH_CMD: cmd= %d, allowed = %d retries= %d \n", $cmd, $cmd->allowed, $cmd->retries)
                        printf("SCSI_DISPATCH_CMD: scsi_cmnd= %d (host,channel,id,lun)= (%d, %d, %d, %d) \n", $cmd, $cmd->device->host->host_no, $cmd->device->channel, $cmd->device->id, $cmd->device->lun)
                        printf("SCSI_DISPATCH_CMD: execname=%s, pexecname=%s\n", execname(), pexecname())
                }
        }
}

[dwalkes]

@CharmingYang0 I haven't tried to use this tool for several years or on any recent kernel, but what I would do if I were trying to use it is to research when this function was removed from the kernel, using strategies like https://stackoverflow.com/questions/12591247/find-when-line-was-deleted on the linux kernel git repo, then look at the commits which replaced it and try to understand how the function was replaced in other uses.

[CharmingYang0]

@dwalkes
Got it.
The solution will be post here once I figure it out.

[CharmingYang0]

@dwalkes
Below case I want to confirm with you.

STEPS

1. Keep temporary_rerr.stp running in inode mode which associates to a text file that not cached in memory
2. Read the file via command head -n 5 file.txt

RESULTS
The first 5 line displayed even error found in dmesg and 'scsi_decide_disposition : major=8 minor=xx scmd=xxx' found.

Now that the read error was injected, it's suposed to return some error like 'I/O error' for the head command.
Any idea?

SCSI_DISPATCH_CMD: scmd = 0xffff9bceb4dbb8e8
SCSI_DISPATCH_CMD: command= 0x28
SCSI_DISPATCH_CMD: access_type= 2, transfer_direction= 2
SCSI_DISPATCH_CMD: major= 8 minor= 32
SCSI_DISPATCH_CMD: flag(0:LBA, 1:inode)= 1
SCSI_DISPATCH_CMD: start sector= 2240
SCSI_DISPATCH_CMD: req bufflen= 32768
SCSI_DISPATCH_CMD: inode= 132
SCSI_DISPATCH_CMD: [7]=0 [8]=64
SCSI_DISPATCH_CMD: cmd-retries = 0 entire-retry =0 

SCSI_DISPATCH_CMD: scsi_cmnd= 0xffff9bceb4dbb8e8, allowed= 5 retries= 0
SCSI_DISPATCH_CMD: (host,channel,id,lun)= (0, 0, 2, 0)
scsi_decide_disposition : major=8 minor=32 scmd=0xffff9bceb4dbb8e8 
scsi_end_request: scmd = 0xffff9bceb4dbb8e8

[root@VERIFY ~]# dmesg -T|grep error
[Fri Oct 16 18:07:20 2020] sd 0:0:2:0: [sdc] tag#78 Add. Sense: Unrecovered read error - auto reallocate failed
[Fri Oct 16 18:07:20 2020] blk_update_request: critical medium error, dev sdc, sector 2240 op 0x0:(READ) flags 0x80700 phys_seg 6 prio class 0

[dwalkes]

@CharmingYang0 my first guess is that something is happening in filesystem caching which means the head command is still succeeding. Do you get the expected content from file.txt output from the head command?

I'd suggest start by verifying you can see the error using ddpt with iflag=sync to make sure you aren't dealing with filesystem caching. It looks like this will succeed based on the dmesg output.

Assuming this works, you could experiment with unmounting the filesystem between the error injection and the read step to force filesystem cache flush. There are probably other ways to accomplish this through open synchronized I/O as well.