From 207f637f3782c8023bd2903efb0ef2efdcbd97ef Mon Sep 17 00:00:00 2001 From: Laszlo Valko Date: Sun, 6 Nov 2022 10:02:04 +0100 Subject: [PATCH] Implemented mksrc command --- .gitignore | 1 + README.md | 1 + config/00defaults.yaml | 27 +++++ config/01package.yaml | 4 + configuration.py | 145 +++++++++++++++++++----- ktool | 8 ++ tool.py | 249 +++++++++++++++++++++++++++++++++++++++++ 7 files changed, 405 insertions(+), 30 deletions(-) create mode 100644 config/01package.yaml create mode 100644 tool.py diff --git a/.gitignore b/.gitignore index bada32d..38196e1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *~ __pycache__ archive +distribution diff --git a/README.md b/README.md index 9bb7a5e..1ee7f28 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ ktool mksrc - load kernel source - add .config - make prepare +- check if .config changed -> store it - store prepared kernel source ```bash diff --git a/config/00defaults.yaml b/config/00defaults.yaml index cf7e4f3..730ac69 100644 --- a/config/00defaults.yaml +++ b/config/00defaults.yaml @@ -6,3 +6,30 @@ distdir: distribution # kernel series to use (latest or x.y) kernels: latest + +# kernel package config selection +package: mygentoo + +# build parameters +ebuild: +# cleanup after build + cleanup: no +# portage build location +## prefix: /var/tmp/portage +# format to use to store source, must exist under 'formats' + format: tarxz + +# format descriptions +# extension: extension to use for this format +# archive: command to create archive +# ${SRCDIR}: source directory to archive +# ${DESTDIR}: destination directory for archive file +# ${DESTFILENAME}: destination filename for archive file +# extract: command to extract archive +# ${SRCFILE}: source file for archive +# ${DESTDIR}: destination directory to extract to +formats: + tarxz: + extension: tar.xz + archive: 'set -o pipefail && mkdir -p "${DESTDIR}" && tar cfC - "${SRCDIR}" . | xz -c - > "${DESTDIR}/.${DESTFILENAME}" && mv -f "${DESTDIR}/.${DESTFILENAME}" "${DESTDIR}/${DESTFILENAME}"' + extract: 'set -o pipefail && mkdir -p "${DESTDIR}" && xz -cd "${SRCFILE}" | tar xfC - "${DESTDIR}"' diff --git a/config/01package.yaml b/config/01package.yaml new file mode 100644 index 0000000..2d8c8c3 --- /dev/null +++ b/config/01package.yaml @@ -0,0 +1,4 @@ +# kernel package configs +packages: + mygentoo: + pkg: sys-kernel/mygentoo-sources diff --git a/configuration.py b/configuration.py index b9e6a20..1e9fa43 100644 --- a/configuration.py +++ b/configuration.py @@ -3,6 +3,7 @@ import os import sys import getopt +import json from yaml import load, YAMLError try: from yaml import CSafeLoader as SafeLoader @@ -10,6 +11,51 @@ except ImportError: from yaml import SafeLoader from pathlib import Path +def merge_config(src, dest): + for key, value in src.items(): + if isinstance(value, dict): + node = dest.setdefault(key, {}) + merge_config(value, node) + else: + dest[key] = value + return dest + +def parse_config_set(params, a): + a = a.strip() + if a.startswith('['): + return "config parameters are not of list type" + if a.startswith('{'): + try: + value = json.loads(a) + merge_config(value, params) + return None + except json.JSONDecodeError as e: + return "failed to decode JSON struct: %s" % (e) + + parts = a.partition('=') + key = parts[0].strip() + operator = parts[1].strip() + value = parts[2].strip() + if (operator != '='): + return "format required: key=jsonvalue or JSON struct" + if (key == ''): + return "format required: key=jsonvalue; key may not be an empty string" + try: + value = json.loads(value) + except json.JSONDecodeError as e: + return "failed to decode JSON value: %s" % (e) + + key_list = key.split('.') + p = params + while True: + key = key_list.pop(0) + more = len(key_list) > 0 + if not more: + p[key] = value + break + p = params.setdefault(key, {}) + return None + class Config(): def __init__(self, scriptname, progver, commands): scriptname = os.path.realpath(scriptname) @@ -21,6 +67,7 @@ class Config(): self.help = False self.version = False self.verbose = False + self.force = False self.config = "config" self.params = {} # command @@ -40,7 +87,9 @@ Options: -h, --help Print this help -V, --version Print version number -c, --config dir Specify config directory to read [""" + self.config + """] - -s, --set key=value Override config parameter key=value + -s, --set key=jsonvalue Override config parameter 'key' with JSON format value + -s, --set jsonstruct Override config parameters providing JSON struct + -f, --force Force recreating result -v, --verbose Print verbose messages """ sys.stderr.write(usage) @@ -49,8 +98,8 @@ Options: def parse(self, argv): try: - (opt, args) = getopt.gnu_getopt(argv[1:], "hVvc:s:", [ - "help", "version", "verbose", "config=", "set=" + (opt, args) = getopt.gnu_getopt(argv[1:], "hVvc:s:f", [ + "help", "version", "verbose", "config=", "set=", "force" ]) except getopt.GetoptError as e: sys.stderr.write("Error: %s\n" % str(e)) @@ -63,23 +112,21 @@ Options: self.version = True elif o == '-v' or o == "--verbose": self.verbose = True + elif o == '-f' or o == "--force": + self.force = True elif o == '-c' or o == "--config": if (a == ''): - sys.stderr.write("Error: option " + o + " requires a non-empty argument\n") + sys.stderr.write("Error: option %s requires a non-empty argument\n" % (o)) self.usage(2) self.config = a elif o == '-s' or o == "--set": - parts = a.partition('=') - if (parts[1] != '='): - sys.stderr.write("Error: option " + o + " requires argument in the format key=value\n") + error = parse_config_set(self.params, a) + if error: + sys.stderr.write("Error: option %s argument is invalid: %s\n" % (o, error)) self.usage(2) - if (parts[0] == ''): - sys.stderr.write("Error: option " + o + " requires argument in the format key=value; key may not be an empty string\n") - self.usage(2) - self.params[parts[0]] = parts[2] if self.version: - sys.stdout.write("Version: " + self.progver + "\n") + sys.stdout.write("Version: %s\n" % (self.progver)) if self.help: self.usage(None) if self.version or self.help: @@ -89,52 +136,90 @@ Options: for c in args: if self.cmd != None: - sys.stderr.write("Multiple commands specified: " + self.cmd + " vs " + c + "\n") + sys.stderr.write("Multiple commands specified: %s vs %s\n" % (self.cmd, c)) self.usage(2) for cmd, _ in self.commands: if c == cmd: self.cmd = c break if self.cmd == None: - sys.stderr.write("Unknown command specified: " + c + "\n") + sys.stderr.write("Unknown command specified: %s\n" % (c)) self.usage(2) if self.cmd == None: sys.stderr.write("Missing command\n") self.usage(2) - for k, v in self.params.items(): - self.config_params[k] = v + merge_config(self.params, self.config_params) if self.verbose: sys.stdout.write("Config parameters:\n") for k, v in self.config_params.items(): - sys.stdout.write(" " + k + "=" + str(v) + "\n") - - def merge_config(self, src, dest): - for key, value in src.items(): - if isinstance(value, dict): - node = dest.setdefault(key, {}) - self.merge_config(value, node) - else: - dest[key] = value - return dest + sys.stdout.write(" %s=%s\n" % (k, str(v))) def parse_file(self, filepath): try: obj = load(open(filepath, 'r'), SafeLoader) - self.merge_config(obj, self.config_params) + merge_config(obj, self.config_params) except YAMLError as exc: sys.stdout.write("Error: reading configuration file failed: %s" % (exc)) def load_config(self): configdir = self.config if self.config.startswith('/') else self.basedir + '/' + self.config if self.verbose: - sys.stdout.write("Reading config directory " + configdir + "\n") + sys.stdout.write("Reading config directory %s\n" % (configdir)) if not Path(configdir).is_dir(): - sys.stderr.write("Warning: config directory " + configdir + " does not exist\n") + sys.stderr.write("Warning: config directory %s does not exist\n" % (configdir)) return with os.scandir(configdir) as it: for entry in it: if not entry.name.startswith('.') and (entry.name.endswith('.yml') or entry.name.endswith('.yaml')) and entry.is_file(): if self.verbose: - sys.stdout.write("Parsing config file " + entry.path + "\n") + sys.stdout.write("Parsing config file %s\n" % (entry.path)) self.parse_file(entry.path) + + def get_config_param(self, path, mandatory = False, default = None, type = 'str'): + missing = False + lpath = path.split('.') + index = 0 + d = self.config_params + while index < len(lpath): + key = lpath[index] + index += 1 + if d == None: + missing = True + break + if not isinstance(d, dict): + missing = True + break + if key not in d: + missing = True + break + d = d[key] + if missing: + if mandatory: + sys.stdout.write("Configuration setting '%s' missing\n" % (path)) + return default + if type == 'str': + return str(d) + if type == 'boolean': + if isinstance(d, (int, float)): + return int(d) != 0 + value = str(d) + if value == '' or value == '0' or value == 'no' or value == 'No' or value == 'false' or value == 'False': + return False + return True + if type == 'int': + try: + return int(d) + except ValueError as e: + sys.stdout.write("Configuration '%s' is not an integer\n" % (path)) + return default + if type == 'float': + try: + return float(d) + except ValueError as e: + sys.stdout.write("Configuration '%s' is not a float\n" % (path)) + return default + except OverflowError as e: + sys.stdout.write("Configuration '%s' is overflows a float\n" % (path)) + return default + return d diff --git a/ktool b/ktool index 1682a83..7f8a3ee 100755 --- a/ktool +++ b/ktool @@ -4,6 +4,7 @@ import os import sys +import tool import progver from configuration import Config @@ -17,3 +18,10 @@ config = Config(__file__, progver.VERSION, [ ['install', 'Install kernel'] ]) config.parse(sys.argv) +rc = None +if config.cmd == 'mksrc': + rc = tool.make_source(config) +elif config.cmd == 'prepare': + rc = tool.prepare_source(config) +if rc: + sys.exit(rc) diff --git a/tool.py b/tool.py new file mode 100644 index 0000000..f4db666 --- /dev/null +++ b/tool.py @@ -0,0 +1,249 @@ +# -*- coding: UTF-8 -*- + +import os +import re +import sys +import subprocess + +DISTDIR_DEFAULT = 'distribution' +BUILD_PREFIX_DEFAULT = '/var/tmp/portage' +PARSE_EBUILD_FILENAME = re.compile("^.*/([^/]+)/([^/]+)/([^/]+)\.ebuild$") + +# run command +def run_command(config, task_name, argv, env = None): + sys.stdout.write("Running command '%s' to %s\n" % (' '.join(argv), task_name)) + try: + result = subprocess.run(argv, stdout=subprocess.PIPE, env=env) + if config.verbose: + sys.stdout.write("Command output: %s\n" % (result.stdout.decode('utf-8'))) + if result.returncode != 0: + sys.stdout.write("Failed to %s: exit code %d\n" % (task_name, result.returncode)) + return None + return result.stdout.decode('utf-8') + except Exception as e: + sys.stdout.write("Failed to %s: %s\n" % (task_name, e)) + return None + +# collect source making configuration +def get_source_package_config(config, context): + config_pkg_name = config.get_config_param('package', mandatory=True) + if config_pkg_name == None: + return 3 + config_pkg_id = 'packages.%s.pkg' % (config_pkg_name) + package_name = config.get_config_param(config_pkg_id, mandatory=True) + if package_name == None: + return 3 + + build_clean = config.get_config_param('ebuild.cleanup', default=True, type='boolean') + context['build_clean'] = build_clean + + version_wanted = config.get_config_param('kernels', default='latest') + if version_wanted == 'latest': + version_wanted = '' + package_to_check = package_name if version_wanted == '' else '%s-%s*' % (package_name, version_wanted) + build_prefix = config.get_config_param('ebuild.prefix', default=BUILD_PREFIX_DEFAULT) + distdir = config.get_config_param('distdir', default=DISTDIR_DEFAULT) + context['package_to_check'] = package_to_check + context['build_prefix'] = build_prefix + context['distdir'] = distdir + + build_source_format = config.get_config_param('ebuild.format', mandatory=True) + if build_source_format == None: + return 3 + build_source_extension = config.get_config_param('formats.%s.extension' % (build_source_format), mandatory=True) + if build_source_extension == None: + return 3 + build_source_archive = config.get_config_param('formats.%s.archive' % (build_source_format), mandatory=True) + if build_source_archive == None: + return 3 + build_source_extract = config.get_config_param('formats.%s.extract' % (build_source_format), mandatory=True) + if build_source_extract == None: + return 3 + context['build_source_format'] = build_source_format + context['build_source_extension'] = build_source_extension + context['build_source_archive'] = build_source_archive + context['build_source_extract'] = build_source_extract + return 0 + +# determine source ebuild and version info +def get_source_ebuild_info(config, context): + result = run_command(config, "get ebuild file for package %s" % (context['package_to_check']), ['equery', 'which', context['package_to_check']]) + if result == None: + return 3 + found_ebuild_filename = result.strip() + + m = PARSE_EBUILD_FILENAME.match(found_ebuild_filename) + if not m: + sys.stdout.write("Could not parse ebuild file name: %s\n" % (found_ebuild_filename)) + return 3 + found_package_group = m.group(1) + found_package_name = m.group(2) + found_package_plus_version = m.group(3) + if not found_package_plus_version.startswith(found_package_name) or found_package_plus_version[len(found_package_name):len(found_package_name)+1] != '-': + sys.stdout.write("Inconsistent package name in ebuild file name: %s\n" % (found_ebuild_filename)) + return 3 + found_version = found_package_plus_version[len(found_package_name) + 1:] + found_package_fullname = '%s/%s-%s' % (found_package_group, found_package_name, found_version) + sys.stdout.write("Package version found: %s\n" % (found_package_fullname)) + + context['found_ebuild_filename'] = found_ebuild_filename + context['found_package_fullname'] = found_package_fullname + context['build_image_dir'] = "%s/%s/image" % (context['build_prefix'], found_package_fullname) + context['build_env_file'] = "%s/%s/temp/environment" % (context['build_prefix'], found_package_fullname) + return 0 + +# extract build variables +def extract_source_variables(config, context): + vars = dict() + for variable in ('KV_FULL', 'KV_MAJOR', 'KV_MINOR', 'KV_PATCH'): + result = run_command(config, "extract kernel version string from %s" % (context['build_env_file']), ['env', '-i', 'bash', '-c', "source %s && echo \"${%s}\"" % (context['build_env_file'], variable)]) + if result == None: + return "Could not extract kernel variable %s" % (variable) + vars[variable] = result.strip() + if vars[variable] == '': + return "Could not extract kernel variable %s" % (variable) + + kernel_version_string = vars['KV_FULL'] + context['kernel_version_string'] = kernel_version_string + kernel_version_major = vars['KV_MAJOR'] + context['kernel_version_major'] = kernel_version_major + kernel_version_minor = vars['KV_MINOR'] + context['kernel_version_minor'] = kernel_version_minor + kernel_version_patch = vars['KV_PATCH'] + context['kernel_version_patch'] = kernel_version_patch + kernel_series = "%s.%s" % (kernel_version_major, kernel_version_minor) + context['kernel_series'] = kernel_series + kernel_series_distdir = "%s/%s" % (context['distdir'], kernel_series) + context['kernel_series_distdir'] = kernel_series_distdir + context['build_kernel_dir'] = "%s/usr/src/linux-%s" % (context['build_image_dir'], kernel_version_string) + source_archive_filename = "%s.%s" % (kernel_version_string, context['build_source_extension']) + context['source_archive_filename'] = source_archive_filename + context['source_archive_dist'] = "%s/%s" % (kernel_series_distdir, source_archive_filename) + sys.stdout.write("Kernel version: %s.%s.%s full: %s\n" % (kernel_version_major, kernel_version_minor, kernel_version_patch, kernel_version_string)) + + return "" + +# determine if source build is needed +def check_source_build_needed(config, context): + sys.stdout.write("Checking build environment file: %s\n" % (context['build_env_file'])) + if not os.path.isfile(context['build_env_file']): + if config.force: + sys.stdout.write("No kernel build environment file found but forced to rebuild source image anyway\n") + return True + sys.stdout.write("No kernel build environment file found\n") + result = run_command(config, "create kernel build environment file from ebuild %s" % (context['found_ebuild_filename']), ['ebuild', context['found_ebuild_filename'], 'clean', 'setup']) + if result == None: + return True + if not os.path.isfile(context['build_env_file']): + sys.stdout.write("No kernel build environment file found after setup\n") + return True + else: + if config.force: + sys.stdout.write("Existing kernel build environment file found but forced to rebuild source image\n") + return True + error = extract_source_variables(config, context) + if error: + sys.stdout.write("%s\n" % (error)) + return True + if os.path.isfile(context['source_archive_dist']): + return False + sys.stdout.write("Checking build image location: %s\n" % (context['build_image_dir'])) + if not os.path.isdir(context['build_image_dir']): + sys.stdout.write("No kernel source image found\n") + return True + sys.stdout.write("Checking build kernel directory: %s\n" % (context['build_kernel_dir'])) + if not os.path.isdir(context['build_kernel_dir']): + sys.stdout.write("No kernel directory found\n") + return True + sys.stdout.write("Existing kernel directory found\n") + return False + +# determine if making build archive is needed +def check_build_archive_needed(config, context): + if os.path.isfile(context['source_archive_dist']): + if os.path.isdir(context['build_kernel_dir']): + try: + dest_mtime = os.path.getmtime(context['source_archive_dist']) + except OSError as e: + return True + try: + src_mtime = os.path.getmtime(context['build_kernel_dir']) + except OSError as e: + return True + sys.stdout.write("%s vs %s\n" % (src_mtime, dest_mtime)) + if src_mtime > dest_mtime: + sys.stdout.write("Existing source archive older than built image\n") + return True + sys.stdout.write("Existing source archive found: %s\n" % (context['source_archive_dist'])) + return False + return True + +# create archive for built source +def make_build_source_archive(config, context): + sys.stdout.write("Using build kernel directory: %s\n" % (context['build_kernel_dir'])) + env = dict(os.environ) + env['SRCDIR'] = context['build_kernel_dir'] + env['DESTDIR'] = context['kernel_series_distdir'] + env['DESTFILENAME'] = context['source_archive_filename'] + result = run_command(config, "create archive for kernel source %s" % (context['source_archive_dist']), ['bash', '-c', context['build_source_archive']], env) + if result == None: + return 3 + if not os.path.isfile(context['source_archive_dist']): + sys.stdout.write("Kernel source archive file not found after creation\n") + return 3 + return 0 + +# build kernel source +def build_source(config, context): + result = run_command(config, "create kernel source from ebuild %s" % (context['found_ebuild_filename']), ['ebuild', context['found_ebuild_filename'], 'clean', 'install']) + if result == None: + return 3 + if not os.path.isfile(context['build_env_file']): + sys.stdout.write("No kernel build environment file found after ebuild command\n") + return 3 + error = extract_source_variables(config, context) + if error: + sys.stdout.write("%s after ebuild command\n" % (error)) + return 3 + if not os.path.isdir(context['build_image_dir']): + sys.stdout.write("No kernel source image found after ebuild command\n") + return 3 + if not os.path.isdir(context['build_kernel_dir']): + sys.stdout.write("No kernel directory found after ebuild command\n") + return 3 + return 0 + +# clear kernel source built +def clear_source(config, context): + run_command(config, "clear kernel source for ebuild %s" % (context['found_ebuild_filename']), ['ebuild', context['found_ebuild_filename'], 'clean']) + return 0 + +# make source +def make_source(config): + context = dict() + rc = get_source_package_config(config, context) + if rc: + return rc + rc = get_source_ebuild_info(config, context) + if rc: + return rc + build = check_source_build_needed(config, context) + if build: + rc = build_source(config, context) + if rc: + return rc + make = check_build_archive_needed(config, context) + if make: + rc = make_build_source_archive(config, context) + if rc: + return rc + if context['build_clean']: + rc = clear_source(config, context) + if rc: + return rc + return None + +# prepare source +def prepare_source(config): + contetx = dict() + return None