Archive for the ‘programming’ Category

android: kernel: example: hung

November 8, 2015

This post demonstrate a kernel module which demonstrates some conditions in which a user space process is hung at kernel space.

example program
kernel-module-hung

software of testing device
LA.BF64.1.1-06510-8×94.0 with Android 5.0.0_r2(LRX21M) and Linux kernel 3.10.49.

hardware of testing device
Architecture arm-v8 cortex-53.

build this module

$ export ARCH=arm64
$ export CROSS_COMPILE=aarch64-linux-android-
$ make KERNEL_PATH=${android_source}/kernel O=${android_source}/out/target/product/${project}/obj/KERNEL_OBJ

install and uninstall this module

$ adb push hung.ko /data
$ adb shell insmod /data/hung.ko
$ adb shell rmmod hung

how this module works

  • hung at mutex deadlock
  • $ adb shell "echo 1 > /sys/kernel/debug/hung/mutex"
    $ adb shell ps
    root      604   1     5848   376   ffffffff 0042c77c S /sbin/adbd
    root      6118  604   2868   1004  fc0080f8 9c4ef8fc D /system/bin/sh
    $ adb shell cat /proc/6118/stack
    [<0000000000000000>] __switch_to+0x90/0x9c
    [<0000000000000000>] mutex_store+0x44/0x5c [hung]
    [<0000000000000000>] vfs_write+0xcc/0x178
    [<0000000000000000>] SyS_write+0x44/0x74
    [<0000000000000000>] cpu_switch_to+0x48/0x4c
    [<0000000000000000>] 0xffffffffffffffff
    
  • hung at msleep for 1000 seconds
  • $ adb shell "echo 1 > /sys/kernel/debug/hung/msleep"
    $ adb shell ps
    root      604   1     5848   376   ffffffff 0042c77c S /sbin/adbd
    root      6891  604   2868   1000  0022bf84 b5d5b8fc D /system/bin/sh
    $ adb shell cat /proc/6891/stack
    [<0000000000000000>] __switch_to+0x90/0x9c
    [<0000000000000000>] msleep+0x14/0x24
    [<0000000000000000>] msleep_store+0x38/0x50 [hung]
    [<0000000000000000>] vfs_write+0xcc/0x178
    [<0000000000000000>] SyS_write+0x44/0x74
    [<0000000000000000>] cpu_switch_to+0x48/0x4c
    [<0000000000000000>] 0xffffffffffffffff
    
  • hung at mdelay for 1000 seconds
  • $ adb shell "echo 1 > /sys/kernel/debug/hung/mdelay"
    $ adb shell ps
    root      604   1     5848   376   ffffffff 0042c77c S /sbin/adbd
    root      6595  604   2868   1000  00000000 b24778fc R /system/bin/sh
    $ adb shell cat /proc/6595/stack
    [<0000000000000000>] __switch_to+0x90/0x9c
    [<0000000000000000>] sched_account_irqtime+0xe0/0xfc
    [<0000000000000000>] irqtime_account_irq+0xec/0x108
    [<0000000000000000>] __do_softirq+0x244/0x284
    [<0000000000000000>] do_softirq+0x40/0x54
    [<0000000000000000>] uncached_logk_pc+0xdc/0xf8
    [<0000000000000000>] gic_handle_irq+0xb0/0xcc
    [<0000000000000000>] el1_irq+0x64/0xd4
    [<0000000000000000>] __delay+0x18/0x38
    [<0000000000000000>] __const_udelay+0x20/0x2c
    [<0000000000000000>] $x+0x48/0x64 [hung]
    [<0000000000000000>] vfs_write+0xcc/0x178
    [<0000000000000000>] SyS_write+0x44/0x74
    [<0000000000000000>] cpu_switch_to+0x48/0x4c
    [<0000000000000000>] 0xffffffffffffffff
    

    conclusion
    This post demonstrate a kernel module which demonstrates some conditions in which a user space process is hung at kernel space. Under these condition, these process are hung until kernel restarts.

    Advertisements

    android: kernel: example: stack

    November 8, 2015

    This post demonstrate a kernel module which implements a stack with kernel list_head data structure.

    example program
    kernel-module-stack

    software of testing device
    LA.BF64.1.1-06510-8×94.0 with Android 5.0.0_r2(LRX21M) and Linux kernel 3.10.49.

    hardware of testing device
    Architecture arm-v8 cortex-53.

    build this module

    $ export ARCH=arm64
    $ export CROSS_COMPILE=aarch64-linux-android-
    $ make KERNEL_PATH=${android_source}/kernel O=${android_source}/out/target/product/${project}/obj/KERNEL_OBJ
    

    install and uninstall this module

    $ adb push stack.ko /data
    $ adb shell insmod /data/stack.ko
    $ adb shell rmmod stack
    

    how this module works

    $ adb shell insmod /data/stack.ko
    $ adb shell 'echo 1 > /sys/kernel/debug/stack/push'
    $ adb shell 'echo 2 > /sys/kernel/debug/stack/push'
    $ adb shell 'echo 3 > /sys/kernel/debug/stack/push'
    $ adb shell 'cat /sys/kernel/debug/stack/show'
    3 2 1
    $ adb shell 'echo 4 > /sys/kernel/debug/stack/push'
    $ adb shell 'echo 5 > /sys/kernel/debug/stack/push'
    $ adb shell 'cat /sys/kernel/debug/stack/show'
    5 4 3 2 1
    $ adb shell 'echo 1 > /sys/kernel/debug/stack/pop'
    $ adb shell 'echo 1 > /sys/kernel/debug/stack/pop'
    $ adb shell 'cat /sys/kernel/debug/stack/show'
    3 2 1
    $ adb shell 'echo 1 > /sys/kernel/debug/stack/pop'
    $ adb shell 'echo 1 > /sys/kernel/debug/stack/pop'
    $ adb shell 'cat /sys/kernel/debug/stack/show'
    1
    

    declare a stack
    The internal implementation of the stack is a double linked list.

    struct val_node {
    	struct list_head list;
    	int val;
    };
    
    static struct list_head val_list = LIST_HEAD_INIT(val_list);
    

    implementation of stack operations
    All linked list operations are done by manipulating the embedded list_head of each node.

    static struct val_node *val_node_alloc(int val)
    {
    	struct val_node *node;
    
    	node = kmalloc(sizeof(*node), GFP_KERNEL);
    	INIT_LIST_HEAD(&node->list);
    	node->val = val;
    
    	return node;
    }
    
    static void val_list_push(int val)
    {
    	struct val_node *node = val_node_alloc(val);
    
    	list_add(&node->list, &val_list);
    }
    
    static void val_list_pop(void)
    {
    	struct list_head *list = val_list.next;
    	struct val_node *node = list_entry(list, struct val_node, list);
    
    	list_del(list);
    	kfree(node);
    }
    

    release the linked list while this module is uninstalled
    Release the underlying linked list.

    static void val_list_release(void)
    {
    	struct val_node *node, *n;
    
    	list_for_each_entry_safe(node, n, &val_list, list)
    		kfree(node);
    }
    
    static void __exit linked_list_exit(void)
    {
    	debugfs_remove(show_dentry);
    	debugfs_remove(add_head_dentry);
    	debugfs_remove(add_tail_dentry);
    	debugfs_remove(del_head_dentry);
    	debugfs_remove(del_tail_dentry);
    	debugfs_remove(linked_list_dentry);
    	val_list_release();
    }
    

    conclusion
    This post demonstrate a kernel module which shows how to declare a stack, implementation of the stack operations, and release of the stack while this module is uninstalled.

    android: kernel: example: linked_list

    November 8, 2015

    This post demonstrate a kernel module which implements linked_list operations with kernel list_head data structure.

    example program
    kernel-module-linked_list

    software of testing device
    LA.BF64.1.1-06510-8×94.0 with Android 5.0.0_r2(LRX21M) and Linux kernel 3.10.49.

    hardware of testing device
    Architecture arm-v8 cortex-53.

    build this module

    $ export ARCH=arm64
    $ export CROSS_COMPILE=aarch64-linux-android-
    $ make KERNEL_PATH=${android_source}/kernel O=${android_source}/out/target/product/${project}/obj/KERNEL_OBJ
    

    install and uninstall this module

    $ adb push linked_list.ko /data
    $ adb shell insmod /data/linked_list.ko
    $ adb shell rmmod linked_list
    

    how this module works

    $ adb shell "echo 1 > /sys/kernel/debug/linked_list/add_head"
    $ adb shell "echo 2 > /sys/kernel/debug/linked_list/add_head"
    $ adb shell "echo 3 > /sys/kernel/debug/linked_list/add_head"
    $ adb shell "cat /sys/kernel/debug/linked_list/show"
    3 2 1
    $ adb shell "echo 4 > /sys/kernel/debug/linked_list/add_tail"
    $ adb shell "echo 5 > /sys/kernel/debug/linked_list/add_tail"
    $ adb shell "cat /sys/kernel/debug/linked_list/show"
    3 2 1 4 5
    $ adb shell "echo 1 > /sys/kernel/debug/linked_list/del_head"
    $ adb shell "echo 1 > /sys/kernel/debug/linked_list/del_head"
    $ adb shell "cat /sys/kernel/debug/linked_list/show"
    1 4 5
    $ adb shell "echo 1 > /sys/kernel/debug/linked_list/del_tail"
    $ adb shell "echo 1 > /sys/kernel/debug/linked_list/del_tail"
    $ adb shell "cat /sys/kernel/debug/linked_list/show"
    1
    

    declare a linked list
    In Linux, all nodes in a linked list are connected with list_head which is embedded in each linked list node. We only need to declare the embedded list_head to represent a linked list.

    struct val_node {
    	struct list_head list;
    	int val;
    };
    
    static struct list_head val_list = LIST_HEAD_INIT(val_list);
    

    implementation of linked list operations
    All linked list operations are done by manipulating the embedded list_head of each node.

    static struct val_node *val_node_alloc(int val)
    {
    	struct val_node *node;
    
    	node = kmalloc(sizeof(*node), GFP_KERNEL);
    	INIT_LIST_HEAD(&node->list);
    	node->val = val;
    
    	return node;
    }
    
    static void val_list_add_head(int val)
    {
    	struct val_node *node = val_node_alloc(val);
    
    	list_add(&node->list, &val_list);
    }
    
    static void val_list_add_tail(int val)
    {
    	struct val_node *node = val_node_alloc(val);
    
    	list_add_tail(&node->list, &val_list);
    }
    
    static void val_list_del_head(void)
    {
    	struct list_head *list = val_list.next;
    	struct val_node *node = list_entry(list, struct val_node, list);
    
    	list_del(list);
    	kfree(node);
    }
    
    static void val_list_del_tail(void)
    {
    	struct list_head *list = val_list.prev;
    	struct val_node *node = list_entry(list, struct val_node, list);
    
    	list_del(list);
    	kfree(node);
    }
    

    release the linked list while this module is uninstalled

    static void val_list_release(void)
    {
    	struct val_node *node, *n;
    
    	list_for_each_entry_safe(node, n, &val_list, list)
    		kfree(node);
    }
    
    static void __exit linked_list_exit(void)
    {
    	debugfs_remove(show_dentry);
    	debugfs_remove(add_head_dentry);
    	debugfs_remove(add_tail_dentry);
    	debugfs_remove(del_head_dentry);
    	debugfs_remove(del_tail_dentry);
    	debugfs_remove(linked_list_dentry);
    	val_list_release();
    }
    

    conclusion
    This post demonstrate a kernel module which shows how to declare a linked list, implementation of linked_list operations, and release of linked list while this module is uninstalled.

    android: kernel: example: debugfs hello

    November 7, 2015

    This post demonstrate a kernel module which uses debugfs to export information to user space.

    example program
    kernel-module-debugfs_hello

    software of testing device
    LA.BF64.1.1-06510-8×94.0 with Android 5.0.0_r2(LRX21M) and Linux kernel 3.10.49.

    hardware of testing device
    Architecture arm-v8 cortex-53.

    build this module

    $ export ARCH=arm64
    $ export CROSS_COMPILE=aarch64-linux-android-
    $ make KERNEL_PATH=${android_source}/kernel O=${android_source}/out/target/product/${project}/obj/KERNEL_OBJ
    

    install and uninstall this module

    $ adb push debugfs_hello.ko /data
    $ adb shell insmod /data/debugfs_hello.ko
    $ adb shell rmmod debugfs_hello
    

    how this module works

    $ adb shell cat /sys/kernel/debug/hello
    debugfs hello world
    

    create & remove debugfs file node
    debugfs_create_file() and debugfs_remove() are the two APIs to create and remove debugfs file nodes.

    
    static struct dentry *hello_dentry;
    
    static int __init debugfs_hello_init(void)
    {
    	hello_dentry = debugfs_create_file("hello", S_IRUGO | S_IWUGO, NULL, NULL, &hello_fops);
    
    	if (!hello_dentry) {
    		pr_err("failed to create /sys/kernel/debug/hello\n");
    		return -ENOMEM;
    	}
    
    	return 0;
    }
    
    static void __exit debugfs_hello_exit(void)
    {
    	debugfs_remove(hello_dentry);
    }
    

    use seq_file to show data to user space
    This module use seq_file to show data to user space. While using seq_file, be careful of memory leakage explained in kernel: patch: memory leakage: single_open.

    static int hello_show(struct seq_file *m, void *v)
    {
    	seq_printf(m, "debugfs hello world\n");
    
    	return 0;
    }
    
    static int hello_open(struct inode *inode, struct file *file)
    {
    	return single_open(file, hello_show, inode->i_private);
    }
    
    static const struct file_operations hello_fops = {
    	.open = hello_open,
    	.read = seq_read,
    	.llseek = seq_lseek,
    	.release = single_release,
    };
    

    conclusion
    This post demonstrate how to create a debugfs file node. This file node uses seq_file to write data from kernel to user space.

    android: kernel: example: procfs hello

    November 3, 2015

    This post demonstrate a kernel module which uses procfs to export information to user space.

    example program
    kernel-module-procfs_hello.

    software of testing device
    LA.BF64.1.1-06510-8×94.0 with Android 5.0.0_r2(LRX21M) and Linux kernel 3.10.49.

    hardware of testing device
    Architecture arm-v8 cortex-53.

    build this module

    $ export ARCH=arm64
    $ export CROSS_COMPILE=aarch64-linux-android-
    $ make KERNEL_PATH=${android_source}/kernel O=${android_source}/out/target/product/${project}/obj/KERNEL_OBJ
    

    install and uninstall this module

    $ adb push procfs_hello.ko /data
    $ adb shell insmod /data/procfs_hello.ko
    $ adb shell rmmod procfs_hello
    

    how this module works

    $ adb shell cat /proc/hello
    procfs hello world
    

    create & remove procfs file node
    proc_create() and proc_remove() are the two APIs to create and remove debugfs file nodes.

    
    tatic struct proc_dir_entry *hello_dentry;
    
    static int __init procfs_hello_init(void)
    {
    	hello_dentry = proc_create("hello", 0, NULL, &hello_fops);
    
    	if (!hello_dentry) {
    		pr_err("failed to create /proc/hello\n");
    		return -ENOMEM;
    	}
    
    	return 0;
    }
    
    static void __exit procfs_hello_exit(void)
    {
    	proc_remove(hello_dentry);
    }
    

    use seq_file to show data to user space
    This module use seq_file to show data to user space. While using seq_file, be careful of memory leakage explained in kernel: patch: memory leakage: single_open.

    static int hello_show(struct seq_file *m, void *v)
    {
    	seq_printf(m, "procfs hello world\n");
    
    	return 0;
    }
    
    static int hello_open(struct inode *inode, struct file *file)
    {
    	return single_open(file, hello_show, inode->i_private);
    }
    
    static const struct file_operations hello_fops = {
    	.open = hello_open,
    	.read = seq_read,
    	.llseek = seq_lseek,
    	.release = single_release,
    };
    

    conclusion
    This post demonstrate how to create a procfs file node. This file node uses seq_file to write data from kernel to user space.

    android: kernel: hello kernel module

    November 3, 2015

    This post is to demonstrate a simple kernel module example.

    example program
    kernel-module-hello.

    software of testing device
    LA.BF64.1.1-06510-8×94.0 with Android 5.0.0_r2(LRX21M) and Linux kernel 3.10.49.

    hardware of testing device
    Architecture arm-v8 cortex-53.

    build this module

    $ export ARCH=arm64
    $ export CROSS_COMPILE=aarch64-linux-android-
    $ make KERNEL_PATH=${android_source}/kernel O=${android_source}/out/target/product/${project}/obj/KERNEL_OBJ
    

    install this module

    $ adb push hello.ko /data
    $ adb shell insmod /data/hello.ko
    $ adb shell rmmod hello
    

    log of installing and uninstalling this module

    $ adb shell dmesg | grep hello
    <6>[24489.720958] hello_init
    <6>[24497.100490] hello_exit
    

    conclusion
    A simple hello kernel module.

    android: init: console service

    September 23, 2015

    In android: what is console , we discuss what is console. In this post, we discusses what console service is. The reference source code here is CodeAuora LA.BR.1.3.1-09730-8952.0.

    $ repo init -u git://codeaurora.org/quic/la/platform/manifest -b refs/tags/LA.BR.1.3.1-09730-8952.0 -m LA.BR.1.3.1-09730-8952.0.xml
    $ repo sync -cd -j32
    

    where is console service declared
    Console is a service defined in init.rc. This service is a shell which is disabled by default. This service sets console flag.

    service console /system/bin/sh
        class core
        console
        disabled
        user shell
        group shell log
        seclabel u:r:shell:s0
    

    when does console service start
    init process starts this service when property property:ro.debuggable=1.

    on property:ro.debuggable=1
        start console
    

    the standard input/output of console service
    Since the console service sets console flag. init process will connect the standard input/output to uart console.

    void service_start(struct service *svc, const char *dynamic_args)
    {
        struct stat s;
        pid_t pid;
        int needs_console;
    ......
        needs_console = (svc->flags & SVC_CONSOLE) ? 1 : 0;
        if (needs_console && (!have_console)) {
            ERROR("service '%s' requires console\n", svc->name);
            svc->flags |= SVC_DISABLED;
            return;
        }
    ......
            if (needs_console) {
                setsid();
                open_console();
    }
    
    static void open_console()
    {
        int fd;
        if ((fd = open(console_name, O_RDWR)) < 0) {
            fd = open("/dev/null", O_RDWR);
        }
        ioctl(fd, TIOCSCTTY, 0);
        dup2(fd, 0);
        dup2(fd, 1);
        dup2(fd, 2);
        close(fd);
    }
    

    debug with uart console
    printk driver prints kernel log into all consoles including uart console. Therefore, we could read kernel log from uart cable at runtime. Since console services is a shell whose standard input/output is connected to uart console, we could type in all commands in console as in adb shell. The results of shell commands are displayed in uart console.

    If adb shell doesn’t work, it’s very hard to debug why device couldn’t boot successfully. If uart console and console service are both enabled, we could type in logcat in uart console and see android logcat logs from uart console directly.

    In this case, we typed in logcat in console uart and found that surfaceflinger service repeatedly crashed and caused zygote and frameworks to restart also.

    [   38.049192] ipa-wan ipa_wwan_ioctl:1311 dev(rmnet_data3) register to IPA
    [   38.061083] ipa-wan ipa_wwan_ioctl:1311 dev(rmnet_data4) register to IPA
    logcat
    [   38.088650] ipa-wan ipa_wwan_ioctl:1311 dev(rmnet_data5) register to IPA
    [   38.103813] ipa-wan ipa_wwan_ioctl:1311 dev(rmnet_data6) register to IPA
    --------- beginning of main
    W/hld     (  430): version 1.1
    W/hld     (  430): trigger_hdf_i[   38.116226] ipa-wan ipa_wwan_ioctl:1311 dev(rmnet_data7) register to IPA
    ......
    I/DEBUG   (  495): Revision: '0'
    I/DEBUG   (  495): ABI: 'arm64'
    I/DEBUG   (  495): pid: 2245, tid: 2245, name: surfaceflinger  >>> /system/bin/surfaceflinger <<<
    I/DEBUG   (  495): signal 11 (SIGSEGV), code 1 (SEGV_MAPERR), fault addr 0x0
    I/DEBUG   (  495):     x0   00000055935f4860  x1   0000000000000000  x2   0000000000000000  x3   00000[   62.696430] init: process 'usb_uicc_daemon', pid 522 exited
    07fa48e3000
    I/DEBUG   (  495):     x4   0000007fa48e3000  x5   0000007fa4b39038  x6   0000000000000001  x7   6566667562656d61
    [   62.708979] init: process 'surfaceflinger', pid 2594 exited
    I/DEBUG   (  495):     x8   0000000000000000  x9   0000000000000000  x10  0000000000000018  x11  0000000000000001
    I/DEBUG   (  495):     x12  0000000000000001  x13  0000007fa4bd8e88  x14  000000559354e000  x15  0000007fa4bd8ed0
    I/DEBUG   (  495):     x16  0000007fa48e3588  x17  0000007fa4b196cc  x18  0000000000000000  x19  00000055936b6950
    I/DEBUG   (  495):     x20  0000000000000000  x21  00000055936b2600  x22  0000007fa489d46c  x23  0000007fa489d5dc
    I/DEBUG   (  495):     x24  0000007fa4aaa016  x25  0[   62.759338] init: process 'surfaceflinger' killing any children in process group
    000007fa4aaa000  x26  00000055936b6950  x27  0000000000000000
    I/DEBUG   (  495):     x28  0000000000000000  x29  0000007fe769d860  x30  0000007fa4a87d28
    I/DEBUG   (  495):     sp   0000007fe769d830  pc   0000007fa4a8996c  pstate 0000000060000000
    D/Netd    ( 2100): execIptables: -t filter -D OUTPUT -j fw_OUTPUT -w res=1
    D/Netd    ( 2100): execIptables: -t filter -F fw_OUTPUT -w 
    I/DEBUG   (  495): 
    I/DEBUG   (  495): backtrace:
    I/DEBUG   (  495):     #00 pc 000000000003996c  /system/lib64/libsurfaceflinger.so
    I/DEBUG   (  495):     #01 pc 0000000000037d24  /system/lib64/libsurfaceflinger.so
    I/DEBUG   (  495):     #02 pc 0000[   62.824307] init: service 'zygote' is being killed
    000000029c70  /system/lib64/libsurfaceflinger.so (android::SurfaceFlinger::init()+508)
    I/DEBUG   (  495):     #03 pc 0000000000000ecc  /system/bin/surfaceflinger
    I/DEBUG   (  495):     #04 pc 00000000000138f4  /system/lib64/libc.so (__libc_init+100)
    I/DEBUG   (  495):     #05 pc 000000000000102c  /system/bin/surfaceflinger
    

    conclusion
    In this post, we discuss what is console service, how it is declared in init.rc, how it is created by init process. Finally, we discuss how developer could tap into console service to get logcat log in uart console to debug boot failure issues.


    %d bloggers like this: