kernel_headers: Fix headers not detected in incremental builds

Currently the Soong logic doesn't kick in if a change updates
headers but doesn't update gen_headers_arm*.bp files for
incremental builds. Fix the issue by making inputs to the
kernel_headers.py script as variables instead of a list.

Change-Id: I9737f6d655848458bfda70b8c6f4d9aa896a98fc
Signed-off-by: Rishabh Bhatnagar <rishabhb@codeaurora.org>

gen_headers_arm.bp

@@ -1,7 +1,20 @@
 // ***** DO NOT EDIT *****
 // This file is generated by kernel_headers.py
+gen_headers_srcs_arm = [
+    "arch/arm/include/uapi/asm/Kbuild",
+    "include/uapi/asm-generic/Kbuild",
+    "Makefile",
+    "arch/arm/tools/syscall.tbl",
+    "include/uapi/**/*.h",
+    "arch/arm/include/uapi/**/*.h",
+    "techpack/*/include/uapi/*/**/*.h",
+]
 
-gen_headers_arm = [
+gen_headers_exclude_srcs_arm = [
+    "include/uapi/linux/a.out.h",
+]
+
+gen_headers_out_arm = [
 
     // Matching generated-y:
 
@@ -938,7 +951,7 @@ gen_headers_arm = [
     "asm/types.h",
     "asm/unistd.h",
 
-    // From techpack/*
+    // From techpack/*/include/uapi/*/**/*.h
 
     "audio/linux/avtimer.h",
     "audio/linux/msm_audio.h",
@@ -998,7 +1011,8 @@ genrule {
     // checked later to ensure that it matches the checked-
     // in version (this file).
     name: "qti_generate_gen_headers_arm",
-    srcs: ["arch/arm/include/uapi/asm/Kbuild", "include/uapi/asm-generic/Kbuild", "arch/arm/include/uapi/**/*.h"],
+    srcs: gen_headers_srcs_arm,
+    exclude_srcs: gen_headers_exclude_srcs_arm,
     tool_files: ["kernel_headers.py"],
     cmd: "python3 $(location kernel_headers.py) " +
         kernel_headers_verbose +
@@ -1006,8 +1020,10 @@ genrule {
         "--gen_dir $(genDir) " +
         "--arch_asm_kbuild $(location arch/arm/include/uapi/asm/Kbuild) " +
         "--arch_include_uapi $(locations arch/arm/include/uapi/**/*.h) " +
+        "--techpack_include_uapi $(locations techpack/*/include/uapi/*/**/*.h) " +
         "--asm_generic_kbuild $(location include/uapi/asm-generic/Kbuild) " +
-        "blueprints",
+        "blueprints " +
+        "# $(in)",
     out: ["gen_headers_arm.bp"],
 }
 
@@ -1018,25 +1034,18 @@ genrule {
         "kernel_headers.py",
         "arch/arm/tools/syscallhdr.sh",
     ],
-    srcs: [
+    srcs: gen_headers_srcs_arm +[
-        "arch/arm/include/uapi/asm/Kbuild",
-        "include/uapi/asm-generic/Kbuild",
         "gen_headers_arm.bp",
         ":qti_generate_gen_headers_arm",
-        "Makefile",
-        "arch/arm/tools/syscall.tbl",
-        "include/uapi/**/*.h",
-        "arch/arm/include/uapi/**/*.h",
-    ],
-    exclude_srcs: [
-        "include/uapi/linux/a.out.h",
     ],
+    exclude_srcs: gen_headers_exclude_srcs_arm,
     cmd: "python3 $(location kernel_headers.py) " +
         kernel_headers_verbose +
         "--header_arch arm " +
         "--gen_dir $(genDir) " +
         "--arch_asm_kbuild $(location arch/arm/include/uapi/asm/Kbuild) " +
         "--arch_include_uapi $(locations arch/arm/include/uapi/**/*.h) " +
+        "--techpack_include_uapi $(locations techpack/*/include/uapi/*/**/*.h) " +
         "--asm_generic_kbuild $(location include/uapi/asm-generic/Kbuild) " +
         "headers " +
         "--old_gen_headers_bp $(location gen_headers_arm.bp) " +
@@ -1046,5 +1055,5 @@ genrule {
         "--arch_syscall_tbl $(location arch/arm/tools/syscall.tbl) " +
         "--headers_install $(location headers_install.sh) " +
         "--include_uapi $(locations include/uapi/**/*.h)",
-    out: ["linux/version.h"] + gen_headers_arm,
+    out: ["linux/version.h"] + gen_headers_out_arm,
 }

gen_headers_arm64.bp

@@ -1,7 +1,20 @@
 // ***** DO NOT EDIT *****
 // This file is generated by kernel_headers.py
 
-gen_headers_arm64 = [
+gen_headers_srcs_arm64 = [
+    "arch/arm64/include/uapi/asm/Kbuild",
+    "include/uapi/asm-generic/Kbuild",
+    "Makefile",
+    "include/uapi/**/*.h",
+    "arch/arm64/include/uapi/**/*.h",
+    "techpack/*/include/uapi/*/**/*.h",
+]
+
+gen_headers_exclude_srcs_arm64 = [
+    "include/uapi/linux/a.out.h",
+]
+
+gen_headers_out_arm64 = [
 
     // Matching generic-y:
 
@@ -934,7 +947,7 @@ gen_headers_arm64 = [
     "asm/ucontext.h",
     "asm/unistd.h",
 
-    // From techpack/*
+    // From techpack/*/include/uapi/*/**/*.h
 
     "audio/linux/avtimer.h",
     "audio/linux/msm_audio.h",
@@ -994,7 +1007,8 @@ genrule {
     // checked later to ensure that it matches the checked-
     // in version (this file).
     name: "qti_generate_gen_headers_arm64",
-    srcs: ["arch/arm64/include/uapi/asm/Kbuild", "include/uapi/asm-generic/Kbuild", "arch/arm64/include/uapi/**/*.h"],
+    srcs: gen_headers_srcs_arm64,
+    exclude_srcs: gen_headers_exclude_srcs_arm64,
     tool_files: ["kernel_headers.py"],
     cmd: "python3 $(location kernel_headers.py) " +
         kernel_headers_verbose +
@@ -1002,8 +1016,10 @@ genrule {
         "--gen_dir $(genDir) " +
         "--arch_asm_kbuild $(location arch/arm64/include/uapi/asm/Kbuild) " +
         "--arch_include_uapi $(locations arch/arm64/include/uapi/**/*.h) " +
+        "--techpack_include_uapi $(locations techpack/*/include/uapi/*/**/*.h) " +
         "--asm_generic_kbuild $(location include/uapi/asm-generic/Kbuild) " +
-        "blueprints",
+        "blueprints " +
+        "# $(in)",
     out: ["gen_headers_arm64.bp"],
 }
 
@@ -1013,24 +1029,18 @@ genrule {
     tool_files: [
         "kernel_headers.py",
     ],
-    srcs: [
+    srcs: gen_headers_srcs_arm64 +[
-        "arch/arm64/include/uapi/asm/Kbuild",
-        "include/uapi/asm-generic/Kbuild",
         "gen_headers_arm64.bp",
         ":qti_generate_gen_headers_arm64",
-        "Makefile",
-        "include/uapi/**/*.h",
-        "arch/arm64/include/uapi/**/*.h",
-    ],
-    exclude_srcs: [
-        "include/uapi/linux/a.out.h",
     ],
+    exclude_srcs: gen_headers_exclude_srcs_arm64,
     cmd: "python3 $(location kernel_headers.py) " +
         kernel_headers_verbose +
         "--header_arch arm64 " +
         "--gen_dir $(genDir) " +
         "--arch_asm_kbuild $(location arch/arm64/include/uapi/asm/Kbuild) " +
         "--arch_include_uapi $(locations arch/arm64/include/uapi/**/*.h) " +
+        "--techpack_include_uapi $(locations techpack/*/include/uapi/*/**/*.h) " +
         "--asm_generic_kbuild $(location include/uapi/asm-generic/Kbuild) " +
         "headers " +
         "--old_gen_headers_bp $(location gen_headers_arm64.bp) " +
@@ -1038,5 +1048,5 @@ genrule {
         "--version_makefile $(location Makefile) " +
         "--headers_install $(location headers_install.sh) " +
         "--include_uapi $(locations include/uapi/**/*.h)",
-    out: ["linux/version.h"] + gen_headers_arm64,
+    out: ["linux/version.h"] + gen_headers_out_arm64,
 }

kernel_headers.py

@@ -521,7 +521,7 @@ def gen_blueprints(
   arch_prefix = os.path.join('arch', header_arch, generic_prefix)
   generic_src = os.path.join(generic_prefix, rel_glob)
   arch_src = os.path.join(arch_prefix, rel_glob)
-  techpack_src = 'techpack/*'
+  techpack_src = os.path.join('techpack/*',generic_prefix, '*',rel_glob)
 
   # Excluded sources, architecture specific.
   exclude_srcs = []
@@ -560,7 +560,28 @@ def gen_blueprints(
     f.write('// ***** DO NOT EDIT *****\n')
     f.write('// This file is generated by %s\n' % kernel_headers_py)
     f.write('\n')
-    f.write('gen_headers_%s = [\n' % header_arch)
+    f.write('gen_headers_srcs_%s = [\n' % header_arch)
+    f.write('    "%s",\n' % rel_arch_asm_kbuild)
+    f.write('    "%s",\n' % rel_asm_generic_kbuild)
+    f.write('    "%s",\n' % makefile)
+
+    if header_arch == "arm":
+      f.write('    "%s",\n' % arm_syscall_tbl)
+
+    f.write('    "%s",\n' % generic_src)
+    f.write('    "%s",\n' % arch_src)
+    f.write('    "%s",\n' % techpack_src)
+    f.write(']\n')
+    f.write('\n')
+
+    if exclude_srcs:
+      f.write('gen_headers_exclude_srcs_%s = [\n' % header_arch)
+      for h in exclude_srcs:
+        f.write('    "%s",\n' % os.path.join(generic_prefix, h))
+      f.write(']\n')
+      f.write('\n')
+
+    f.write('gen_headers_out_%s = [\n' % header_arch)
 
     if generated_list:
       f.write('\n')
@@ -615,7 +636,10 @@ def gen_blueprints(
     f.write('    // checked later to ensure that it matches the checked-\n')
     f.write('    // in version (this file).\n')
     f.write('    name: "%s",\n' % gen_blueprints_module_name)
-    f.write('    srcs: ["%s", "%s", "%s"],\n' % (rel_arch_asm_kbuild, rel_asm_generic_kbuild, arch_src))
+    f.write('    srcs: gen_headers_srcs_%s,\n' % header_arch)
+    if exclude_srcs:
+      f.write('    exclude_srcs: gen_headers_exclude_srcs_%s,\n' % header_arch)
+
     f.write('    tool_files: ["kernel_headers.py"],\n')
     f.write('    cmd: "python3 $(location kernel_headers.py) " +\n')
     f.write('        kernel_headers_verbose +\n')
@@ -623,8 +647,10 @@ def gen_blueprints(
     f.write('        "--gen_dir $(genDir) " +\n')
     f.write('        "--arch_asm_kbuild $(location %s) " +\n' % rel_arch_asm_kbuild)
     f.write('        "--arch_include_uapi $(locations %s) " +\n' % arch_src)
+    f.write('        "--techpack_include_uapi $(locations %s) " +\n' % techpack_src)
     f.write('        "--asm_generic_kbuild $(location %s) " +\n' % rel_asm_generic_kbuild)
-    f.write('        "blueprints",\n')
+    f.write('        "blueprints " +\n')
+    f.write('        "# $(in)",\n')
     f.write('    out: ["gen_headers_%s.bp"],\n' % header_arch)
     f.write('}\n')
     f.write('\n')
@@ -639,25 +665,13 @@ def gen_blueprints(
       f.write('        "%s",\n' % arm_syscall_tool)
 
     f.write('    ],\n')
-    f.write('    srcs: [\n')
+    f.write('    srcs: gen_headers_srcs_%s +[\n' % header_arch)
-    f.write('        "%s",\n' % rel_arch_asm_kbuild)
-    f.write('        "%s",\n' % rel_asm_generic_kbuild)
     f.write('        "%s",\n' % old_gen_headers_bp)
     f.write('        ":%s",\n' % gen_blueprints_module_name)
-    f.write('        "%s",\n' % makefile)
-
-    if header_arch == "arm":
-      f.write('        "%s",\n' % arm_syscall_tbl)
-
-    f.write('        "%s",\n' % generic_src)
-    f.write('        "%s",\n' % arch_src)
     f.write('    ],\n')
 
     if exclude_srcs:
-      f.write('    exclude_srcs: [\n')
+      f.write('    exclude_srcs: gen_headers_exclude_srcs_%s,\n' % header_arch)
-      for h in exclude_srcs:
-        f.write('        "%s",\n' % os.path.join(generic_prefix, h))
-      f.write('    ],\n')
 
     f.write('    cmd: "python3 $(location %s) " +\n' % kernel_headers_py)
     f.write('        kernel_headers_verbose +\n')
@@ -665,6 +679,7 @@ def gen_blueprints(
     f.write('        "--gen_dir $(genDir) " +\n')
     f.write('        "--arch_asm_kbuild $(location %s) " +\n' % rel_arch_asm_kbuild)
     f.write('        "--arch_include_uapi $(locations %s) " +\n' % arch_src)
+    f.write('        "--techpack_include_uapi $(locations %s) " +\n' % techpack_src)
     f.write('        "--asm_generic_kbuild $(location %s) " +\n' % rel_asm_generic_kbuild)
     f.write('        "headers " +\n')
     f.write('        "--old_gen_headers_bp $(location %s) " +\n' % old_gen_headers_bp)
@@ -677,14 +692,14 @@ def gen_blueprints(
 
     f.write('        "--headers_install $(location %s) " +\n' % headers_install_sh)
     f.write('        "--include_uapi $(locations %s)",\n' % generic_src)
-    f.write('    out: ["linux/version.h"] + gen_headers_%s,\n' % header_arch)
+    f.write('    out: ["linux/version.h"] + gen_headers_out_%s,\n' % header_arch)
     f.write('}\n')
 
     return 0
 
 def parse_bp_for_headers(file_name, headers):
   parsing_headers = False
-  pattern = re.compile("gen_headers_[a-zA-Z0-9]+\s*=\s*\[\s*")
+  pattern = re.compile("gen_headers_out_[a-zA-Z0-9]+\s*=\s*\[\s*")
   with open(file_name, 'r') as f:
     for line in f:
       line = line.strip()
@@ -877,6 +892,11 @@ def main():
       required=True,
       nargs='*',
       help='The list of arch/<arch>/include/uapi header files.')
+  parser.add_argument(
+      '--techpack_include_uapi',
+      required=True,
+      nargs='*',
+      help='The list of techpack/*/include/uapi header files.')
 
   # The modes.
 
@@ -948,14 +968,11 @@ def main():
   if args.verbose:
     print('module_dir [%s]' % module_dir)
 
-  # Get list of techpack uapi headers to be exported from techpack/* directories.
-  techpack_uapi = extract_techpack_uapi_headers(args.verbose, module_dir)
-
 
   if args.mode == 'blueprints':
     return gen_blueprints(
         args.verbose, args.header_arch, args.gen_dir, args.arch_asm_kbuild,
-        args.asm_generic_kbuild, module_dir, rel_arch_asm_kbuild, rel_asm_generic_kbuild, args.arch_include_uapi, techpack_uapi)
+        args.asm_generic_kbuild, module_dir, rel_arch_asm_kbuild, rel_asm_generic_kbuild, args.arch_include_uapi, args.techpack_include_uapi)
 
   if args.mode == 'headers':
     if args.verbose:
@@ -970,7 +987,7 @@ def main():
         args.verbose, args.header_arch, args.gen_dir, args.arch_asm_kbuild,
         args.asm_generic_kbuild, module_dir, args.old_gen_headers_bp, args.new_gen_headers_bp,
         args.version_makefile, args.arch_syscall_tool, args.arch_syscall_tbl,
-        args.headers_install, args.include_uapi, args.arch_include_uapi, techpack_uapi)
+        args.headers_install, args.include_uapi, args.arch_include_uapi, args.techpack_include_uapi)
 
   print('error: unknown mode: %s' % args.mode)
   return 1