""" Execute pylint and show results to CodeSonar. """ import argparse import os from shlex import split as cmdline2list from subprocess import list2cmdline import sys from tempfile import mkstemp import gtr from gtr.globmatch import iglob, globescape sys.path.append(os.path.join(gtr.gthome(), 'third-party', 'pylint-sarif')) import pylint2sarif PYTHON_SOURCE_EXTS = ('.py',) class RespFileArgumentParser(argparse.ArgumentParser): """ Extend basic ArgumentParser to parse arguments from response files. """ def convert_arg_line_to_args(self, arg_line): return cmdline2list(arg_line) class Pylint2SarifArguments: """ Class to contain parsed commandline arguments for pylint2sarif. """ def __init__(self): self.disable_glob = True self.inputs = [] self.inputs_file = None self.sarif_output = None self.base_dir = None def main(argv, stdio): # flags for import_sarif.py: INCLUDE_FLAG = '-include-sources' EXCLUDE_FLAG = '-exclude-sources' EXCLUDE_SIGIL = '!' ANALYZER_ARG_FLAG = '-X' linesep = '\n' stderr = stdio[2] cwd = os.getcwd() parser = RespFileArgumentParser( fromfile_prefix_chars='@', description='Run Pylint and import the results to CodeSonar.') parser.add_argument( 'analyzer_inputs', metavar='SOURCE', nargs='*', help='Python module file or package directory to analyze.') parser.add_argument( '-C', '-directory', metavar='DIR', dest='base_dir', help='Change to directory DIR before analyzing.') parser.add_argument( '-sarif-output', dest='sarif_output', help='Name of file to save analysis results in SARIF format. Can be used for diagnostics.') parser.add_argument( '-pylint-output', dest='analyzer_output', help='Name of file to save raw analysis results from pylint. Can be used for diagnostics.') parser.add_argument( '-include-sources', metavar='PATTERN', dest='source_args', action='append', type=(lambda s: (INCLUDE_FLAG, s)), help="File pattern for source files to include when importing.") parser.add_argument( '-exclude-sources', metavar='PATTERN', dest='source_args', action='append', type=(lambda s: (EXCLUDE_FLAG, s)), help="File pattern for source files to exclude when importing.") parser.add_argument( '-source-max-bytes', dest='source_max_size', type=int, help="The maximum size in bytes for any included source file.") parser.add_argument( ANALYZER_ARG_FLAG, dest='analyzer_args', action='append', help=f'Arguments to pass to pylint. Arguments should be prefixed and separated by the same character. E.g. "{ANALYZER_ARG_FLAG},-j,4" or (equivalently) "{ANALYZER_ARG_FLAG}+-j+4"') arg_obj, unknown_args = parser.parse_known_args(argv[1:]) for arg in unknown_args: if arg.startswith(ANALYZER_ARG_FLAG): arg_obj.analyzer_args.append(arg[len(ANALYZER_ARG_FLAG):]) else: stderr.write(f'Error: unrecognized argument: {arg}\n') return 2 analyzer_args = [] for arg in arg_obj.analyzer_args or (): if not arg: continue sep = arg[0] arg = arg[len(sep):] args = arg.split(sep) analyzer_args.extend(args) source_args = arg_obj.source_args or [] analyzer_inputs = arg_obj.analyzer_inputs or [] source_max_size = arg_obj.source_max_size sarif_output = arg_obj.sarif_output base_dir = arg_obj.base_dir source_exts = PYTHON_SOURCE_EXTS analyzer_output = arg_obj.analyzer_output if not source_args and not analyzer_inputs: parser.print_usage(stderr) stderr.write("Error: An analysis target is required. Please provide a SOURCE argument or use -include-sources to specify a pattern.\n") return 2 if base_dir: base_dir = os.path.normpath(base_dir) if not base_dir or base_dir == os.path.curdir: base_dir = None # If there are no analyzer_inputs, # then get the list of source files from the source_args. # If there are analyzer inputs, # then only use source_args to filter the files that will be imported into CodeSonar. analyzer_inputs2 = list(analyzer_inputs) if not analyzer_inputs2: analyzer_patterns = [ pat if flag == INCLUDE_FLAG else EXCLUDE_SIGIL + pat for (flag, pat) in source_args] analyzer_inputs2.extend(iglob(analyzer_patterns, filesonly=True, cwd=base_dir)) # Create source file patterns for analyzer inputs (e.g. python package directories) # which we will provide to cs-import.py via import_sarif.py. # Prepend the patterns to the patterns specified with include/exclude options: source_args2 = [] for path in analyzer_inputs: # source_args must be relative to cwd (i.e. joined with base_dir) path2 = os.path.join(base_dir or '', path) if os.path.isdir(path2): # Use normpath to remove initial './' if it exists. # This can help to ensure compatibility with exclusion patterns. source_args2.extend(( ( INCLUDE_FLAG, os.path.normpath(os.path.join( globescape(path2), '**', '*' + ext)), ) for ext in source_exts)) else: source_args2.append( ( INCLUDE_FLAG, globescape(path2), )) for flag, source_pattern in source_args: if base_dir and base_dir != os.path.curdir: source_pattern = os.path.join(base_dir, source_pattern) source_args2.append((flag, source_pattern)) source_args = source_args2 analyzer_inputs = analyzer_inputs2 if not analyzer_inputs: stderr.write( 'Error: No source files matched the analysis source arguments.' ' If you are using the -C option to specify a directory,' ' ensure input argument paths are relative to that directory.') stderr.write(linesep) return 1 # Check if the source patterns actually match anything. # If they don't match anything, then assume user made a commandline error. source_patterns = [ pat if flag == INCLUDE_FLAG else EXCLUDE_SIGIL + pat for (flag, pat) in source_args] for _ in iglob(source_patterns or (), filesonly=True): break else: stderr.write( 'Error: No source files matched the analysis source arguments.' ' If you are using the -C option to specify a working directory,' ' ensure input argument paths are relative to the working directory.') stderr.write(linesep) return 1 # TODO save temporary SARIF into prj_files dir. # Note that Pylint2Sarif.run_pylint() will create its own temp file. temp_sarif_output = None if not sarif_output: fd, temp_sarif_output = mkstemp(prefix='codesonar.python_scan.', suffix='.sarif', dir='.') os.close(fd) sarif_output = temp_sarif_output try: p2s_inputs = list(analyzer_args) p2s_inputs.extend(analyzer_inputs) p2s_args = Pylint2SarifArguments() p2s_args.base_dir = base_dir or None p2s_args.sarif_output = os.path.join(cwd, sarif_output) p2s_args.inputs = p2s_inputs p2s = pylint2sarif.Pylint2Sarif(p2s_args) if analyzer_output: p2s.tmpfile = analyzer_output p2s.run_pylint() sarif_argv = [argv[0], sarif_output] if source_max_size: sarif_argv.extend(('-source-max-bytes', str(source_max_size))) if base_dir: # pylint2sarif is expected to produce absolute URIs, # but provide -path-base anyway just to be safe: sarif_argv.extend(('-path-base', base_dir)) for flag, pattern in source_args: sarif_argv.append(flag) sarif_argv.append(pattern) import import_sarif stderr.write(list2cmdline(sarif_argv)) stderr.write(linesep) stderr.flush() returncode = import_sarif.main(sarif_argv) finally: if temp_sarif_output: os.remove(temp_sarif_output) return returncode if __name__ == '__main__': sys.exit(main(sys.argv, (sys.stdin, sys.stdout, sys.stderr)))