scripts: snapshot of kernel build utility scripts
This is a snapshot of build scripts taken as of msm-4.19 commit <3fc21416733e696> (Merge "ARM: dts: msm: Add dpdm regulator on kona"). Change-Id: I3a60f8563227de4e6f59d5abcca7c8edd2c8c12f Signed-off-by: Raghavendra Rao Ananta <rananta@codeaurora.org>
This commit is contained in:
Raghavendra Rao Ananta
357
scripts/build-all.py
Executable file
357
scripts/build-all.py
Executable file
@@ -0,0 +1,357 @@
|
||||
#! /usr/bin/env python2
|
||||
# SPDX-License-Identifier: GPL-2.0-only
|
||||
# Copyright (c) 2009-2015, 2017-19, The Linux Foundation. All rights reserved.
|
||||
|
||||
# Build the kernel for all targets using the Android build environment.
|
||||
|
||||
from collections import namedtuple
|
||||
import glob
|
||||
from optparse import OptionParser
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import Queue
|
||||
|
||||
version = 'build-all.py, version 1.99'
|
||||
|
||||
build_dir = '../all-kernels'
|
||||
make_command = ["vmlinux", "modules", "dtbs"]
|
||||
all_options = {}
|
||||
compile64 = os.environ.get('CROSS_COMPILE64')
|
||||
clang_bin = os.environ.get('CLANG_BIN')
|
||||
|
||||
def error(msg):
|
||||
sys.stderr.write("error: %s\n" % msg)
|
||||
|
||||
def fail(msg):
|
||||
"""Fail with a user-printed message"""
|
||||
error(msg)
|
||||
sys.exit(1)
|
||||
|
||||
if not os.environ.get('CROSS_COMPILE'):
|
||||
fail("CROSS_COMPILE must be set in the environment")
|
||||
|
||||
def check_kernel():
|
||||
"""Ensure that PWD is a kernel directory"""
|
||||
if not os.path.isfile('MAINTAINERS'):
|
||||
fail("This doesn't seem to be a kernel dir")
|
||||
|
||||
def check_build():
|
||||
"""Ensure that the build directory is present."""
|
||||
if not os.path.isdir(build_dir):
|
||||
try:
|
||||
os.makedirs(build_dir)
|
||||
except OSError as exc:
|
||||
if exc.errno == errno.EEXIST:
|
||||
pass
|
||||
else:
|
||||
raise
|
||||
|
||||
failed_targets = []
|
||||
|
||||
BuildResult = namedtuple('BuildResult', ['status', 'messages'])
|
||||
|
||||
class BuildSequence(namedtuple('BuildSequence', ['log_name', 'short_name', 'steps'])):
|
||||
|
||||
def set_width(self, width):
|
||||
self.width = width
|
||||
|
||||
def __enter__(self):
|
||||
self.log = open(self.log_name, 'w')
|
||||
def __exit__(self, type, value, traceback):
|
||||
self.log.close()
|
||||
|
||||
def run(self):
|
||||
self.status = None
|
||||
messages = ["Building: " + self.short_name]
|
||||
def printer(line):
|
||||
text = "[%-*s] %s" % (self.width, self.short_name, line)
|
||||
messages.append(text)
|
||||
self.log.write(text)
|
||||
self.log.write('\n')
|
||||
for step in self.steps:
|
||||
st = step.run(printer)
|
||||
if st:
|
||||
self.status = BuildResult(self.short_name, messages)
|
||||
break
|
||||
if not self.status:
|
||||
self.status = BuildResult(None, messages)
|
||||
|
||||
class BuildTracker:
|
||||
"""Manages all of the steps necessary to perform a build. The
|
||||
build consists of one or more sequences of steps. The different
|
||||
sequences can be processed independently, while the steps within a
|
||||
sequence must be done in order."""
|
||||
|
||||
def __init__(self, parallel_builds):
|
||||
self.sequence = []
|
||||
self.lock = threading.Lock()
|
||||
self.parallel_builds = parallel_builds
|
||||
|
||||
def add_sequence(self, log_name, short_name, steps):
|
||||
self.sequence.append(BuildSequence(log_name, short_name, steps))
|
||||
|
||||
def longest_name(self):
|
||||
longest = 0
|
||||
for seq in self.sequence:
|
||||
longest = max(longest, len(seq.short_name))
|
||||
return longest
|
||||
|
||||
def __repr__(self):
|
||||
return "BuildTracker(%s)" % self.sequence
|
||||
|
||||
def run_child(self, seq):
|
||||
seq.set_width(self.longest)
|
||||
tok = self.build_tokens.get()
|
||||
with self.lock:
|
||||
print "Building:", seq.short_name
|
||||
with seq:
|
||||
seq.run()
|
||||
self.results.put(seq.status)
|
||||
self.build_tokens.put(tok)
|
||||
|
||||
def run(self):
|
||||
self.longest = self.longest_name()
|
||||
self.results = Queue.Queue()
|
||||
children = []
|
||||
errors = []
|
||||
self.build_tokens = Queue.Queue()
|
||||
nthreads = self.parallel_builds
|
||||
print "Building with", nthreads, "threads"
|
||||
for i in range(nthreads):
|
||||
self.build_tokens.put(True)
|
||||
for seq in self.sequence:
|
||||
child = threading.Thread(target=self.run_child, args=[seq])
|
||||
children.append(child)
|
||||
child.start()
|
||||
for child in children:
|
||||
stats = self.results.get()
|
||||
if all_options.verbose:
|
||||
with self.lock:
|
||||
for line in stats.messages:
|
||||
print line
|
||||
sys.stdout.flush()
|
||||
if stats.status:
|
||||
errors.append(stats.status)
|
||||
for child in children:
|
||||
child.join()
|
||||
if errors:
|
||||
fail("\n ".join(["Failed targets:"] + errors))
|
||||
|
||||
class PrintStep:
|
||||
"""A step that just prints a message"""
|
||||
def __init__(self, message):
|
||||
self.message = message
|
||||
|
||||
def run(self, outp):
|
||||
outp(self.message)
|
||||
|
||||
class MkdirStep:
|
||||
"""A step that makes a directory"""
|
||||
def __init__(self, direc):
|
||||
self.direc = direc
|
||||
|
||||
def run(self, outp):
|
||||
outp("mkdir %s" % self.direc)
|
||||
os.mkdir(self.direc)
|
||||
|
||||
class RmtreeStep:
|
||||
def __init__(self, direc):
|
||||
self.direc = direc
|
||||
|
||||
def run(self, outp):
|
||||
outp("rmtree %s" % self.direc)
|
||||
shutil.rmtree(self.direc, ignore_errors=True)
|
||||
|
||||
class CopyfileStep:
|
||||
def __init__(self, src, dest):
|
||||
self.src = src
|
||||
self.dest = dest
|
||||
|
||||
def run(self, outp):
|
||||
outp("cp %s %s" % (self.src, self.dest))
|
||||
shutil.copyfile(self.src, self.dest)
|
||||
|
||||
class ExecStep:
|
||||
def __init__(self, cmd, **kwargs):
|
||||
self.cmd = cmd
|
||||
self.kwargs = kwargs
|
||||
|
||||
def run(self, outp):
|
||||
outp("exec: %s" % (" ".join(self.cmd),))
|
||||
with open('/dev/null', 'r') as devnull:
|
||||
proc = subprocess.Popen(self.cmd, stdin=devnull,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
**self.kwargs)
|
||||
stdout = proc.stdout
|
||||
while True:
|
||||
line = stdout.readline()
|
||||
if not line:
|
||||
break
|
||||
line = line.rstrip('\n')
|
||||
outp(line)
|
||||
result = proc.wait()
|
||||
if result != 0:
|
||||
return ('error', result)
|
||||
else:
|
||||
return None
|
||||
|
||||
class Builder():
|
||||
|
||||
def __init__(self, name, defconfig):
|
||||
self.name = name
|
||||
self.defconfig = defconfig
|
||||
|
||||
self.confname = re.sub('arch/arm[64]*/configs/', '', self.defconfig)
|
||||
|
||||
# Determine if this is a 64-bit target based on the location
|
||||
# of the defconfig.
|
||||
self.make_env = os.environ.copy()
|
||||
if "/arm64/" in defconfig:
|
||||
if compile64:
|
||||
self.make_env['CROSS_COMPILE'] = compile64
|
||||
else:
|
||||
fail("Attempting to build 64-bit, without setting CROSS_COMPILE64")
|
||||
self.make_env['ARCH'] = 'arm64'
|
||||
else:
|
||||
self.make_env['ARCH'] = 'arm'
|
||||
self.make_env['KCONFIG_NOTIMESTAMP'] = 'true'
|
||||
self.log_name = "%s/log-%s.log" % (build_dir, self.name)
|
||||
|
||||
def build(self):
|
||||
steps = []
|
||||
dest_dir = os.path.join(build_dir, self.name)
|
||||
log_name = "%s/log-%s.log" % (build_dir, self.name)
|
||||
steps.append(PrintStep('Building %s in %s log %s' %
|
||||
(self.name, dest_dir, log_name)))
|
||||
if not os.path.isdir(dest_dir):
|
||||
steps.append(MkdirStep(dest_dir))
|
||||
defconfig = self.defconfig
|
||||
dotconfig = '%s/.config' % dest_dir
|
||||
savedefconfig = '%s/defconfig' % dest_dir
|
||||
|
||||
staging_dir = 'install_staging'
|
||||
modi_dir = '%s' % staging_dir
|
||||
hdri_dir = '%s/usr' % staging_dir
|
||||
steps.append(RmtreeStep(os.path.join(dest_dir, staging_dir)))
|
||||
|
||||
steps.append(ExecStep(['make', 'O=%s' % dest_dir,
|
||||
self.confname], env=self.make_env))
|
||||
|
||||
# Build targets can be dependent upon the completion of
|
||||
# previous build targets, so build them one at a time.
|
||||
cmd_line = ['make',
|
||||
'INSTALL_HDR_PATH=%s' % hdri_dir,
|
||||
'INSTALL_MOD_PATH=%s' % modi_dir,
|
||||
'O=%s' % dest_dir,
|
||||
'REAL_CC=%s' % clang_bin]
|
||||
build_targets = []
|
||||
for c in make_command:
|
||||
if re.match(r'^-{1,2}\w', c):
|
||||
cmd_line.append(c)
|
||||
else:
|
||||
build_targets.append(c)
|
||||
for t in build_targets:
|
||||
steps.append(ExecStep(cmd_line + [t], env=self.make_env))
|
||||
|
||||
return steps
|
||||
|
||||
def scan_configs():
|
||||
"""
|
||||
Get the full list of defconfigs appropriate for this tree,
|
||||
except for gki_defconfig.
|
||||
"""
|
||||
names = []
|
||||
for defconfig in glob.glob('arch/arm*/configs/vendor/*_defconfig'):
|
||||
target = os.path.basename(defconfig)[:-10]
|
||||
if 'gki' == target:
|
||||
continue
|
||||
name = target + "-llvm"
|
||||
if 'arch/arm64' in defconfig:
|
||||
name = name + "-64"
|
||||
names.append(Builder(name, defconfig))
|
||||
|
||||
return names
|
||||
|
||||
def build_many(targets):
|
||||
print "Building %d target(s)" % len(targets)
|
||||
|
||||
# To try and make up for the link phase being serial, try to do
|
||||
# two full builds in parallel. Don't do too many because lots of
|
||||
# parallel builds tends to use up available memory rather quickly.
|
||||
parallel = 2
|
||||
if all_options.jobs and all_options.jobs > 1:
|
||||
j = max(all_options.jobs / parallel, 2)
|
||||
make_command.append("-j" + str(j))
|
||||
|
||||
tracker = BuildTracker(parallel)
|
||||
for target in targets:
|
||||
steps = target.build()
|
||||
tracker.add_sequence(target.log_name, target.name, steps)
|
||||
tracker.run()
|
||||
|
||||
def main():
|
||||
global make_command
|
||||
|
||||
check_kernel()
|
||||
check_build()
|
||||
|
||||
configs = scan_configs()
|
||||
|
||||
usage = ("""
|
||||
%prog [options] all -- Build all targets
|
||||
%prog [options] target target ... -- List specific targets
|
||||
""")
|
||||
parser = OptionParser(usage=usage, version=version)
|
||||
parser.add_option('--list', action='store_true',
|
||||
dest='list',
|
||||
help='List available targets')
|
||||
parser.add_option('-v', '--verbose', action='store_true',
|
||||
dest='verbose',
|
||||
help='Output to stdout in addition to log file')
|
||||
parser.add_option('-j', '--jobs', type='int', dest="jobs",
|
||||
help="Number of simultaneous jobs")
|
||||
parser.add_option('-l', '--load-average', type='int',
|
||||
dest='load_average',
|
||||
help="Don't start multiple jobs unless load is below LOAD_AVERAGE")
|
||||
parser.add_option('-k', '--keep-going', action='store_true',
|
||||
dest='keep_going', default=False,
|
||||
help="Keep building other targets if a target fails")
|
||||
parser.add_option('-m', '--make-target', action='append',
|
||||
help='Build the indicated make target (default: %s)' %
|
||||
' '.join(make_command))
|
||||
|
||||
(options, args) = parser.parse_args()
|
||||
global all_options
|
||||
all_options = options
|
||||
|
||||
if options.list:
|
||||
print "Available targets:"
|
||||
for target in configs:
|
||||
print " %s" % target.name
|
||||
sys.exit(0)
|
||||
|
||||
if options.make_target:
|
||||
make_command = options.make_target
|
||||
|
||||
if args == ['all']:
|
||||
build_many(configs)
|
||||
elif len(args) > 0:
|
||||
all_configs = {}
|
||||
for t in configs:
|
||||
all_configs[t.name] = t
|
||||
targets = []
|
||||
for t in args:
|
||||
if t not in all_configs:
|
||||
parser.error("Target '%s' not one of %s" % (t, all_configs.keys()))
|
||||
targets.append(all_configs[t])
|
||||
build_many(targets)
|
||||
else:
|
||||
parser.error("Must specify a target to build, or 'all'")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user