Merge 1fbf373e3dc3d83d4bc56febff8cabe5a34c78cc into 3bb373867917b674265067cbd38b9d252c43d014

scripts/field_report.py

@@ -0,0 +1,288 @@
+#!/usr/bin/env python3
+"""Report on structure types defined in C source files.
+
+This script uses the clang python bindings (``pip3 install --user clang``).
+
+This script only works on code that compiles. If there are any errors,
+this script will just return garbage data (typically missing fields, or
+having 0 values everywhere). The most common cause of failure is missing
+include directories, either for the code you're analyzing or for the
+standard library when cross-compiling.
+"""
+
+# Copyright The Mbed TLS Contributors
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import collections
+import re
+import sys
+from typing import Dict, List, Optional
+
+import clang.cindex #type: ignore
+from clang.cindex import Cursor, SourceLocation, TranslationUnit
+from clang.cindex import CursorKind, TypeKind
+
+from mbedtls_dev import typing_util
+
+
+class SanityCheck(Exception):
+    def __init__(self, msg: str) -> None:
+        super().__init__('Sanity check failed: ' + msg +
+                         '\nThis likely indicates a compilation error.' +
+                         '\nMaybe a missing include directory (-I)?')
+
+
+
+class FieldInfo:
+    """Information about a field of a structure."""
+
+    def __init__(self, node: Cursor) -> None:
+        self.node = node
+        self.lexical_uses = 0
+        self.size = node.type.get_size()
+        self.align = node.type.get_align()
+        # Empirically, offsetof is in bits, not bytes. To make the output
+        # easier to read, convert to bytes (the same unit as size and
+        # alignment), which means that bitfields will be located at their
+        # first byte.
+        self.offset = node.get_field_offsetof() // 8
+
+    def name(self) -> str:
+        return self.node.spelling
+
+    def record_use(self, _node: Cursor) -> None:
+        self.lexical_uses += 1
+
+    def uses(self) -> int:
+        return self.lexical_uses
+
+    def is_indirect(self) -> bool:
+        """Whether access to this field is indirect on ARM Cortex-M0+."""
+        word_offset = self.offset // self.align
+        return word_offset >= 128
+
+    def score(self) -> int:
+        """A field's score is an estimate of its access cost.
+
+        The cost is calculated as the number of instructions needed to
+        access the field on ARM Cortex-M0+, multiplied by the number of uses
+        of the field inside the library.
+        """
+        cost_per_use = 1 + 2 * self.is_indirect()
+        return self.uses() * cost_per_use
+
+
+class Ast:
+    """Abstract representation of the source code."""
+
+    def __init__(self, options) -> None:
+        """Prepare for analysis of C source files."""
+        if options.clang_library_file:
+            clang.cindex.Config.set_library_file(options.clang_library_file)
+        self.parse_options = []
+        if options.target:
+            self.parse_options += ['-target', options.target]
+        for d in options.include:
+            self.parse_options.append('-I' + d)
+        for d in options.define:
+            self.parse_options.append('-D' + d)
+        self.index = clang.cindex.Index.create()
+        self.files = {} #type: Dict[str, TranslationUnit]
+        # fields[TYPE_OR_STRUCT_NAME][FIELD_NAME]
+        self.fields = {} #type: Dict[str, Dict[str, FieldInfo]]
+
+    def load(self, *filenames: str) -> None:
+        """Load the AST of the given source files."""
+        for filename in filenames:
+            self.files[filename] = self.index.parse(filename,
+                                                    self.parse_options)
+
+    def in_interesting_file(self, location: SourceLocation) -> bool:
+        """Whether the given location is in a file that should be analyzed."""
+        if not hasattr(location.file, 'name'):
+            # Some artificial nodes have associated no file name.
+            # Let's hope they're not important.
+            return False
+        return location.file.name in self.files
+
+    def read_field_definitions(self, filenames: List[str]) -> None:
+        """Parse structure field definitions in the given C source files (usually headers)."""
+        self.load(*filenames)
+        for filename in filenames:
+            for node in self.files[filename].cursor.walk_preorder():
+                if not self.in_interesting_file(node.location):
+                    continue
+                if node.kind == CursorKind.FIELD_DECL:
+                    type_name = node.lexical_parent.spelling
+                    # Skip unions
+                    if node.lexical_parent.kind != CursorKind.STRUCT_DECL:
+                        continue
+                    # type_name is the struct name. If there's no struct
+                    # name, see if there's a typedef name, to cope with
+                    # "typedef struct { ... } foo;"
+                    if not type_name:
+                        type_name = node.lexical_parent.type.spelling
+                    # Skip anonymous structs for now.
+                    if '(anonymous ' in type_name:
+                        continue
+                    self.fields.setdefault(type_name, collections.OrderedDict())
+                    self.fields[type_name][node.spelling] = FieldInfo(node)
+
+    @staticmethod
+    def get_underlying_type(typ: clang.cindex.Type) -> Optional[clang.cindex.Type]:
+        """Strip off one level of type indirection.
+
+        Return None if the type is as primitive as can be.
+        """
+        if hasattr(typ, 'get_canonical'):
+            lower = typ.get_canonical()
+            if lower != typ:
+                return lower
+        if typ.kind == TypeKind.POINTER:
+            return typ.get_pointee()
+        if hasattr(typ, 'underlying_typedef_type'):
+            return typ.underlying_typedef_type
+        return None
+
+    QUALIFIERS_RE = re.compile(r'.* ')
+    def get_type_core(self, type: clang.cindex.Type) -> str:
+        # pylint: disable=redefined-builtin
+        """Get the base name of a type, without typedefs, qualifiers or pointers."""
+        core = type
+        lower = type # type: Optional[clang.cindex.Type]
+        while lower:
+            core, lower = lower, self.get_underlying_type(core)
+        # There's no API function to remove qualifiers from a type,
+        # so do it textually. Remove 'const', 'restrict', etc.
+        # Also remove 'struct', so we'll get the struct name from struct
+        # definitions.
+        return re.sub(self.QUALIFIERS_RE, r'', core.spelling)
+
+    def record_field_access(self, node: Cursor) -> None:
+        """Record one location where a field is accessed."""
+        field_name = node.spelling
+        lhs = next(node.get_children())
+        structure_type = self.get_type_core(lhs.type)
+        if structure_type not in self.fields:
+            # This is not a structure defined by the library
+            return
+        self.fields[structure_type][field_name].record_use(node)
+
+    def sanity_check_failed(self, log: Optional[typing_util.Writable],
+                            fmt: str, *args, **kwargs) -> None:
+        #pylint: disable=no-self-use,no-else-raise
+        msg = fmt.format(*args, **kwargs)
+        if log is None:
+            raise SanityCheck(msg)
+        else:
+            log.write('Warning: ' + msg + '\n')
+
+    def sanity_checks(self, log: Optional[typing_util.Writable]) -> None:
+        """If the data looks wrong, signal it.
+
+        If `log` is `None`, signaling means to raise an exception explaining
+        the first failure encountered. Otherwise signaling means calling
+        `log.write` with a message for each failure.
+        """
+        for type_name in sorted(self.fields):
+            for field in self.fields[type_name].values():
+                if field.offset == -1:
+                    self.sanity_check_failed(
+                        log,
+                        'Could not determine offset of {}.{}.',
+                        type_name, field.name())
+
+    def run_analysis(self, files: List[str],
+                     log: Optional[typing_util.Writable] = None) -> None:
+        """Run analyses on the specified files.
+
+        Pass `log` to `sanity_checks`.
+        """
+        self.read_field_definitions(files)
+        self.read_field_usage(files)
+        self.sanity_checks(log)
+
+    def read_field_usage(self, filenames: List[str]) -> None:
+        """Parse field usage in the given C source files."""
+        self.load(*filenames)
+        for filename in filenames:
+            for node in self.files[filename].cursor.walk_preorder():
+                if not self.in_interesting_file(node.location):
+                    continue
+                if node.kind == CursorKind.MEMBER_REF_EXPR:
+                    self.record_field_access(node)
+
+    @staticmethod
+    def report_field(out: typing_util.Writable,
+                     prefix: str, field: FieldInfo) -> None:
+        """Print information about a structure field.
+
+        Format: <type>.<name>,<size>,<alignment>,<offset>,<use_count>,<score>
+        """
+        out.write('{},{},{},{},{},{}\n'.format(
+            prefix + field.node.spelling,
+            field.size, field.align, field.offset,
+            field.uses(), field.score()
+        ))
+
+    def report_fields(self, out: typing_util.Writable,
+                      header=False) -> None:
+        """Print information about fields of structures defined by the library."""
+        if header:
+            out.write('field,size,align,offset,uses,score\n')
+        for type_name in sorted(self.fields):
+            for field in self.fields[type_name].values():
+                self.report_field(out, type_name + '.', field)
+
+    def report(self, options, out: typing_util.Writable) -> None:
+        """Report what this script has to report."""
+        self.report_fields(out,
+                           header=options.csv_header)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--clang-library-file',
+                        help="Alternative location of libclang.so")
+    parser.add_argument('--define', '-D',
+                        action='append',
+                        default=[],
+                        help="Additional C preprocessor definition")
+    parser.add_argument('--include', '-I',
+                        action='append',
+                        default=[],
+                        help="Directory to add to the header include path")
+    parser.add_argument('--no-csv-header',
+                        dest='csv_header', default=True, action='store_false',
+                        help="Omit the CSV header from the output")
+    parser.add_argument('--no-sanity-checks',
+                        dest='sanity_checks', default=True, action='store_false',
+                        help="Bypass sanity checks, print output even if it's suspicious")
+    parser.add_argument('--target', '-t',
+                        help="Target triple to build for (default: native build)")
+    parser.add_argument('files', metavar='FILE', nargs='*',
+                        help="Source files to analyze")
+    options = parser.parse_args()
+    ast = Ast(options)
+    if options.sanity_checks:
+        sanity_log = None
+    else:
+        sanity_log = sys.stderr
+    ast.run_analysis(options.files, log=sanity_log)
+    ast.report(options, sys.stdout)
+
+if __name__ == '__main__':
+    main()

scripts/field_report.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# Usage example:
+# scripts/field_report.sh -t armv6m-unknown-eabi -I build-m0plus/include -I ~/Packages/ARMCompiler6.6/include >fields-baremetal-m0plus.csv
+
+set -eu
+
+script_dir=$(dirname -- "$0")
+lib_dir=.
+if [ ! -d "$lib_dir/library" ] && [ -d "${lib_dir%/*}/library" ]; then
+  lib_dir=${lib_dir%/*}
+fi
+
+"$script_dir/field_report.py" \
+  -DMBEDTLS_ALLOW_PRIVATE_ACCESS \
+  "$@" \
+  -I "$lib_dir/include" \
+  "$lib_dir/include"/*/*.h "$lib_dir/library"/*.[hc]