qca-swiss-army-knife/tools/scripts/ath12k/ath12k-check

#!/usr/bin/env python3
#
# Copyright (c) 2015-2017 Qualcomm Atheros, Inc.
# Copyright (c) 2018-2019, The Linux Foundation. All rights reserved.
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#
# Run 'ath12k-check --help' to see the instructions
#

import subprocess
import os
import logging
import sys
import argparse
import re
import tempfile
import queue
import threading
import string
import hashlib
import shutil

CHECKPATCH_COMMIT = '99b70ece33d87500ef7bee8e32cb99772c45ce14'
CHECKPATCH_MD5SUM = 'b3c97930952745672f3408dabc244843'

GIT_URL = 'https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/plain/scripts/checkpatch.pl?id=%s'

DRIVER_DIR = 'drivers/net/wireless/ath/ath12k/'

FILTER_REGEXP = r'/ath'

# Example: CONFIG_CC_VERSION_TEXT="x86_64-linux-gcc (GCC) 13.2.0"
CONFIG_CC_REGEXP = r'^CONFIG_CC_VERSION_TEXT="(.*)"$'

IGNORE_FILES = []

CHECKPATCH_IGNORE = [
    # some find extra parentheses helpful so ignore the warnings
    'UNNECESSARY_PARENTHESES',
]

CHECKPATCH_OPTS = ['--strict', '-q', '--terse', '--no-summary',
                   '--max-line-length=90', '--show-types']

checkpatch_filter = [
    ('qmi_wlanfw_respond_mem_req_msg_v01', 'LONG_LINE'),
    ('qmi_wlanfw_host_cap_req_msg_v01', 'LONG_LINE'),
    ('qmi_wlfw_qdss_trace_config_download_req_msg_v01_ei', 'LONG_LINE'),
    ('qmi_wlanfw_qdss_trace_config_download_resp_msg_v01_ei', 'LONG_LINE'),

    # workaround for long lines in
    # qmi_wlfw_qdss_trace_config_download_req_msg_v01_ei, for some
    # reason gtags claim they are part of
    # ATH12K_QMI_MAX_CHUNK_SIZE
    ('ATH12K_QMI_MAX_CHUNK_SIZE', 'LONG_LINE'),

    # allow long line copyrights
    # note gtags returns None so search for the literal string
    ('Copyright (c)', 'LONG_LINE_COMMENT'),

    # trace.h has warnings which don't make sense to fix
    ('TRACE_SYSTEM', 'OPEN_ENDED_LINE'),

    # couldn't figure out how to make checkpatch happy with these macros
    ('SVC', 'COMPLEX_MACRO'),
    ('C2S', 'COMPLEX_MACRO'),
    ('C2S', 'MACRO_WITH_FLOW_CONTROL'),

    # this use of volatile should be ok
    ('ath12k_hal_srng_access_begin', 'VOLATILE'),
    ('ath12k_hal_srng_access_end', 'VOLATILE'),
    ('hal_srng', 'VOLATILE'),

    # __ath12k_dbg has to use dev_printk() so that debug_mask always work
    ('__ath12k_dbg', 'PREFER_DEV_LEVEL'),
    ('__ath12k_dbg', 'PREFER_PR_LEVEL'),

    # couldn't figure a way to avoid the reuse warning
    ('for_each_ar', 'MACRO_ARG_REUSE'),
]

sparse_filter = []

# global variables

logger = logging.getLogger('ath12k-check')

threads = 1


class CPWarning():

    def __str__(self):
        return 'CPWarning(%s, %s, %s, %s, %s)' % (self.path, self.lineno,
                                                  self.tag, self.type,
                                                  self.msg)

    def __init__(self):
        self.path = ''
        self.lineno = ''
        self.type = ''
        self.msg = ''
        self.tag = ''


def run_gcc(args):
    # to disable utf-8 from gcc, easier to paste that way
    os.environ['LC_CTYPE'] = 'C'

    cmd = 'find %s -name "*.o" -type f -delete' % (DRIVER_DIR)
    logger.debug('%s' % cmd)
    subprocess.call(cmd, shell=True, universal_newlines=True)

    cmd = ['make', '-k', '-j', str(threads)]

    if not args.no_extra:
        cmd.append('W=1')

    cmd.append(DRIVER_DIR)

    env = os.environ.copy()

    # disable ccache in case it's in use, it's useless as we are
    # compiling only few files and it also breaks GCC's
    # -Wimplicit-fallthrough check
    env['CCACHE_DISABLE'] = '1'

    logger.debug('%s' % cmd)
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                         env=env, universal_newlines=True)
    (stdout, stderr) = p.communicate()

    stderr = stderr.strip()

    if len(stderr) > 0:
        for line in stderr.splitlines():
            match = re.search(FILTER_REGEXP, line)
            if not args.no_filter and not match:
                logger.debug('FILTERED: %s' % line)
                continue

            print(line.strip())

    return p.returncode


def run_sparse(args):
    cmd = ['make', '-k', '-j',
           str(threads), DRIVER_DIR, 'C=2', 'CF="-D__CHECK_ENDIAN__"']
    logger.debug('%s' % cmd)
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                         universal_newlines=True)
    (stdout, stderr) = p.communicate()

    stderr = stderr.strip()

    if len(stderr) > 0:
        for line in stderr.splitlines():
            match = re.search(FILTER_REGEXP, line)
            if not args.no_filter and not match:
                logger.debug('FILTERED: %s' % line)
                continue

            drop = False

            for f in sparse_filter:
                match = re.search(f, line)
                if not args.no_filter and match:
                    logger.debug('sparse_filter: %s' % line)
                    drop = True
                    break

            if not drop:
                print(line.strip())

    return p.returncode


def find_tagname(tag_map, filename, lineno):
    if filename.find('Kconfig') != -1:
        return None

    if filename not in tag_map:
        return None

    # we need the tags sorted per linenumber
    sorted_tags = sorted(tag_map[filename], key=lambda tup: tup[0])

    lineno = int(lineno)

    prev = None

    # find the tag which is in lineno
    for (l, tag) in sorted_tags:
        if l > lineno:
            return prev

        prev = tag

    return None


def parse_checkpatch_warning(line):
    m = re.match(r'(.*?):(\d+): .*?:(.*?): (.*)', line, re.M | re.I)
    result = CPWarning()
    result.path = m.group(1)
    result.lineno = m.group(2)
    result.type = m.group(3)
    result.msg = m.group(4)

    return result


def is_filtered(cpwarning):
    for (tag, type) in checkpatch_filter:
        if cpwarning.tag is not None:
            # the global tool was able to find a tag name for this
            # line so use it directly
            matchobj = re.match(tag, cpwarning.tag)
            if matchobj is None:
                continue

            if cpwarning.type == type:
                return True
        else:
            # the global tool was not able to find a tag name for this
            # line so instead check the actual line in the file
            f = open(cpwarning.path)
            buf = f.read()
            f.close()

            lineno = int(cpwarning.lineno) - 1
            lines = buf.splitlines()
            line = lines[lineno]
            logger.debug('file %r tag %r line %r' % (cpwarning.path, tag, line))

            if tag in line:
                return True

    return False


def get_checkpatch_md5sum():
    path = shutil.which('checkpatch.pl')
    f = open(path, 'rb')
    md5sum = hashlib.md5(f.read()).hexdigest()
    f.close()

    return md5sum

def get_checkpatch_cmdline():
    return ['checkpatch.pl'] + CHECKPATCH_OPTS + \
        ['--ignore', ",".join(CHECKPATCH_IGNORE)]


def run_checkpatch_cmd(args, q, tag_map):
    checkpatch_cmd = get_checkpatch_cmdline() + ['-f']

    while True:
        try:
            f = q.get_nowait()
        except queue.Empty:
            # no more files to check
            break

        cmd = checkpatch_cmd + [f]
        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                             universal_newlines=True)
        (stdoutdata, stderrdata) = p.communicate()

        if stdoutdata is None:
            continue

        lines = stdoutdata.splitlines()
        for line in lines:
            w = parse_checkpatch_warning(line)
            w.tag = find_tagname(tag_map, f, w.lineno)

            if not args.no_filter and is_filtered(w):
                logger.debug('FILTERED: %s' % w)
                continue

            logger.debug(w)
            print('%s:%s: %s' % (w.path, w.lineno, w.msg))

        q.task_done()

# get all driver files which are commited to git, includes Kconfig, Makefile etc
def get_commited_files():
    cmd = ['git', 'ls-tree', '-r', '--name-only', 'HEAD', DRIVER_DIR]
    output = subprocess.check_output(cmd, universal_newlines=True)
    return output.splitlines()

def run_checkpatch(args):
    md5sum = get_checkpatch_md5sum()
    if md5sum != CHECKPATCH_MD5SUM:
        print('WARNING: checkpatch.pl md5sum %s does not match with %s' %
              (md5sum, CHECKPATCH_MD5SUM))

    driver_files = get_commited_files()

    # drop files we need to ignore
    for name in IGNORE_FILES:
        full_name = '%s%s' % (DRIVER_DIR, name)
        if full_name in driver_files:
            driver_files.remove(full_name)

    logger.debug('driver_files: %s' % (driver_files))

    # create global index file
    (fd, tmpfilename) = tempfile.mkstemp()
    f = os.fdopen(fd, 'w')
    f.write('\n'.join(driver_files))
    f.close()

    # FIXME: do we need to call os.close(fd) still?

    cmd = 'gtags -f %s' % (tmpfilename)
    logger.debug('%s' % (cmd))
    output = subprocess.check_output(cmd, shell=True, universal_newlines=True)

    os.remove(tmpfilename)

    # tag_map[FILENAME] = [(start line, tagname)]
    tag_map = {}

    # create tag mapping
    for f in driver_files:
        # global gives an error from Kconfig and Makefile
        if f.endswith('Kconfig') or f.endswith('Makefile'):
            continue

        cmd = 'global -f %s' % (f)
        output = subprocess.check_output(cmd, shell=True, universal_newlines=True)
        lines = output.splitlines()
        for l in lines:
            columns = l.split()
            tagname = columns[0]
            line = int(columns[1])

            if f not in tag_map:
                tag_map[f] = []

            tag_map[f].append((line, tagname))

    q = queue.Queue()

    for f in driver_files:
        q.put(f)

    # run checkpatch for all files
    for i in range(threads):
        t = threading.Thread(
            target=run_checkpatch_cmd, args=(args, q, tag_map))
        t.daemon = True
        t.start()

    q.join()

    return 0

def run_kdoc(args):
    p = subprocess.run(['scripts/kernel-doc', '-Werror', '-none'] +
                       get_commited_files())
    return p.returncode

def show_version(args):
    host_gcc_version = 'not found'
    config_cc_version = 'not found'
    sparse_version = 'not found'
    checkpatch_version = 'not found'
    checkpatch_md5sum = 'N/A'
    gtags_version = 'not found'

    run = subprocess.check_output

    f = open(sys.argv[0], 'rb')
    ath12kcheck_md5sum = hashlib.md5(f.read()).hexdigest()
    f.close()

    try:
        host_gcc_version = run(['gcc', '--version'],
                               universal_newlines=True).splitlines()[0]
    except:
        pass

    try:
        f = open('.config')
        m = re.search(CONFIG_CC_REGEXP, f.read(), re.MULTILINE)
        if m is not None:
            config_cc_version = m.group(1)
        f.close()
    except:
        pass

    try:
        sparse_version = run(['sparse', '--version'],
                             universal_newlines=True).splitlines()[0]
    except:
        pass

    try:
        checkpatch_version = run(['checkpatch.pl', '--version'],
                                 universal_newlines=True).splitlines()[1]
        checkpatch_md5sum = get_checkpatch_md5sum()
    except:
        pass

    try:
        gtags_version = run(['gtags', '--version'],
                            universal_newlines=True).splitlines()[0]
    except:
        pass

    print('ath12k-check (md5sum %s)' % (ath12kcheck_md5sum))
    print()
    print('python:\t\t%s' % (sys.version))
    print('host gcc:\t%s' % (host_gcc_version))
    print('config cc:\t%s' % (config_cc_version))
    print('sparse:\t\t%s' % (sparse_version))
    print('checkpatch.pl:\t%s (md5sum %s)' % (checkpatch_version, checkpatch_md5sum))
    print('gtags:\t\t%s' % (gtags_version))

    sys.exit(0)


def main():
    global threads

    checkpatch_url = GIT_URL % (CHECKPATCH_COMMIT)

    description = '''ath12k source code checker

Runs various tests (gcc, sparse and checkpatch) with filtering
unnecessary warnings away, the goal is to have empty output from the
script.

Run this from the main kernel source directory which is preconfigured
with ath12k enabled. gcc recompilation of ath12k is forced every time,
irrespective if there are any changes in source or not. So this can be
run multiple times and every time the same warnings will appear.

Requirements (all available in $PATH):

* gcc
* sparse
* checkpatch.pl
* gtags (from package global)

'''

    s = '''Installation:

It is assumed below that ~/bin is part of user's $$PATH. If it's not,
replace ~/bin with a suitable directory.

To install or update to the latest version of ath12k-check:

wget https://github.com/qca/qca-swiss-army-knife/raw/master/tools/scripts/ath12k/ath12k-check
cp ath12k-check ~/bin/

It's strongly recommend to use the latest GCC version from crosstool:

https://mirrors.edge.kernel.org/pub/tools/crosstool/

Unpack the compiler for example to /opt/cross/ and in the Linux kernel
sources create a new file called GNUmakefile with contents:

ARCH=x86
CROSS_COMPILE=/opt/cross/gcc-13.2.0-nolibc/x86_64-linux/bin/x86_64-linux-
export ARCH
export CROSS_COMPILE

Now the kernel will be compiled with the compiler from /opt/cross.

Latest development version of sparse is needed, install it manually
from the git repository:

git clone https://git.kernel.org/pub/scm/devel/sparse/sparse.git/
cd sparse
make
cp sparse ~/bin/

As checkpatch is evolving and new tests are added, it's important that
a correct version of checkpatch is used. If a wrong version is used
ath12k-check will warn about that. To download the correct checkpatch
version and install it to ~/bin:

wget -O checkpatch.pl $CHECKPATCH_URL
cp checkpatch.pl ~/bin/

Note that the URL in above wget command is dynamically created and
will change everytime ath12k-check is updated to use a new version of
checkpatch.pl

For gtags program you need to install package named global, for
example in Debian/Ubuntu you can do that using apt:

apt install global

Alternatively (but not recommended!) if you want manually run
checkpatch with the same settings as ath12k-check uses here's the
command line:

$CHECKPATCH_CMDLINE
'''

    checkpatch_cmdline = '%s foo.patch' % ' '.join(get_checkpatch_cmdline())
    epilog = string.Template(s).substitute(CHECKPATCH_URL=checkpatch_url,
                                           CHECKPATCH_CMDLINE=checkpatch_cmdline)

    parser = argparse.ArgumentParser(description=description, epilog=epilog,
                                     formatter_class=argparse.RawTextHelpFormatter)

    parser.add_argument('-d', '--debug', action='store_true',
                        help='enable debug messages')

    parser.add_argument('--fast', action='store_true',
                        help='run only tests which finish in few seconds')

    parser.add_argument('--no-extra', action='store_true',
                        help='Do not run extra checks like W=1')

    parser.add_argument('--no-filter', action='store_true',
                        help='Don\'t filter output with regexp: %r' % (FILTER_REGEXP))

    parser.add_argument('--version', action='store_true',
                        help='Show version information about dependencies')

    args = parser.parse_args()

    timefmt = ''

    if args.debug:
        logger.setLevel(logging.DEBUG)
        timefmt = '%(asctime)s '

    logfmt = '%s%%(levelname)s: %%(message)s' % (timefmt)

    logging.basicConfig(format=logfmt)

    if args.version:
        show_version(args)

    if args.fast:
        gcc = True
        sparse = True
        checkpatch = False
    else:
        gcc = True
        sparse = True
        checkpatch = True

    try:
        cores = subprocess.check_output(['nproc'], universal_newlines=True)
    except OSError as xxx_todo_changeme:
        subprocess.CalledProcessError = xxx_todo_changeme
        cores = '4'
        logger.warning('Failed to run nproc, assuming %s cores' % (cores))

    threads = int(cores) + 2

    logger.debug('threads %d' % (threads))

    if gcc:
        ret = run_gcc(args)
        if ret != 0:
            logger.debug('gcc failed: %d', ret)
            sys.exit(1)

    if sparse:
        ret = run_sparse(args)
        if ret != 0:
            logger.debug('sparse failed: %d', ret)
            sys.exit(2)

    if checkpatch:
        ret = run_checkpatch(args)
        if ret != 0:
            logger.debug('checkpatch failed: %d', ret)
            sys.exit(3)

    ret = run_kdoc(args)
    if ret != 0:
        logger.debug('kernel-doc failed: %d', ret)
        sys.exit(4)

if __name__ == "__main__":
    main()