diff --git a/README.md b/README.md index 1ee7f28..d3690ca 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ - emerge unmerge ```bash -ktool mksrc +ktool mksrc [-s kernels=4.19] ``` ### Prepare kernel source @@ -21,7 +21,7 @@ ktool mksrc - store prepared kernel source ```bash -ktool prepare +ktool prepare [-s kernels=4.19] ``` ### Build kernel source package @@ -74,3 +74,5 @@ ktool mkucodes ```bash ktool install ``` + +## Configuration diff --git a/config/00defaults.yaml b/config/00defaults.yaml index 730ac69..8837fbf 100644 --- a/config/00defaults.yaml +++ b/config/00defaults.yaml @@ -18,6 +18,8 @@ ebuild: ## prefix: /var/tmp/portage # format to use to store source, must exist under 'formats' format: tarxz +# source preparation temp directory + tempdir: /var/tmp/prepare # format descriptions # extension: extension to use for this format diff --git a/ktool b/ktool index 7f8a3ee..0debe08 100755 --- a/ktool +++ b/ktool @@ -6,7 +6,7 @@ import sys import tool import progver -from configuration import Config +from parseconfig import Config config = Config(__file__, progver.VERSION, [ ['mksrc', 'Make kernel source'], diff --git a/lib.py b/lib.py new file mode 100644 index 0000000..13d54e8 --- /dev/null +++ b/lib.py @@ -0,0 +1,21 @@ +# -*- coding: UTF-8 -*- + +import sys +import subprocess +from log import * + +# run command and return stdout or None in case of error +######################## +def run_command(config, task_name, argv, env = None): + log(LOG_INFO, "Running command '%s' to %s" % (' '.join(argv), task_name)) + try: + result = subprocess.run(argv, stdout=subprocess.PIPE, env=env) + if config.verbose: + log(LOG_INFO, "Command output: %s" % (result.stdout.decode('utf-8'))) + if result.returncode != 0: + log(LOG_ERROR, "Failed to %s: exit code %d" % (task_name, result.returncode)) + return None + return result.stdout.decode('utf-8') + except Exception as e: + log(LOG_ERROR, "Failed to %s: %s" % (task_name, e)) + return None diff --git a/log.py b/log.py new file mode 100644 index 0000000..854f0d2 --- /dev/null +++ b/log.py @@ -0,0 +1,25 @@ +# -*- coding: UTF-8 -*- + +import sys + +LOG_FATAL = 0 +LOG_ERROR = 1 +LOG_WARNING = 2 +LOG_INFO = 3 +LOG_DEBUG = 4 + +_log_level = LOG_INFO + +# set log level +######################## +def set_log_level(level): + _log_level = level + +# log messages +######################## +def log(level, message): + if level <= _log_level: + if level >= LOG_ERROR: + sys.stdout.write("%s\n" % (message)) + else: + sys.stderr.write("%s\n" % (message)) diff --git a/configuration.py b/parseconfig.py similarity index 95% rename from configuration.py rename to parseconfig.py index 1e9fa43..9436e84 100644 --- a/configuration.py +++ b/parseconfig.py @@ -11,23 +11,23 @@ except ImportError: from yaml import SafeLoader from pathlib import Path -def merge_config(src, dest): +def _merge_config(src, dest): for key, value in src.items(): if isinstance(value, dict): node = dest.setdefault(key, {}) - merge_config(value, node) + _merge_config(value, node) else: dest[key] = value return dest -def parse_config_set(params, a): +def _parse_config_set(params, a): a = a.strip() if a.startswith('['): return "config parameters are not of list type" if a.startswith('{'): try: value = json.loads(a) - merge_config(value, params) + _merge_config(value, params) return None except json.JSONDecodeError as e: return "failed to decode JSON struct: %s" % (e) @@ -120,7 +120,7 @@ Options: self.usage(2) self.config = a elif o == '-s' or o == "--set": - error = parse_config_set(self.params, a) + error = _parse_config_set(self.params, a) if error: sys.stderr.write("Error: option %s argument is invalid: %s\n" % (o, error)) self.usage(2) @@ -149,7 +149,7 @@ Options: sys.stderr.write("Missing command\n") self.usage(2) - merge_config(self.params, self.config_params) + _merge_config(self.params, self.config_params) if self.verbose: sys.stdout.write("Config parameters:\n") for k, v in self.config_params.items(): @@ -158,7 +158,7 @@ Options: def parse_file(self, filepath): try: obj = load(open(filepath, 'r'), SafeLoader) - merge_config(obj, self.config_params) + _merge_config(obj, self.config_params) except YAMLError as exc: sys.stdout.write("Error: reading configuration file failed: %s" % (exc)) diff --git a/progver.py b/progver.py index 4bbdff8..7e7510c 100644 --- a/progver.py +++ b/progver.py @@ -2,5 +2,4 @@ import sys -this = sys.modules[__name__] -this.VERSION = "0.01" +VERSION = "0.01" diff --git a/tool.py b/tool.py index f4db666..b17d705 100644 --- a/tool.py +++ b/tool.py @@ -3,32 +3,57 @@ import os import re import sys -import subprocess +import platform +import lib +from log import * DISTDIR_DEFAULT = 'distribution' BUILD_PREFIX_DEFAULT = '/var/tmp/portage' +BUILD_TEMPDIR_DEFAULT = '/var/tmp/prepare' PARSE_EBUILD_FILENAME = re.compile("^.*/([^/]+)/([^/]+)/([^/]+)\.ebuild$") -# run command -def run_command(config, task_name, argv, env = None): - sys.stdout.write("Running command '%s' to %s\n" % (' '.join(argv), task_name)) - try: - result = subprocess.run(argv, stdout=subprocess.PIPE, env=env) - if config.verbose: - sys.stdout.write("Command output: %s\n" % (result.stdout.decode('utf-8'))) - if result.returncode != 0: - sys.stdout.write("Failed to %s: exit code %d\n" % (task_name, result.returncode)) - return None - return result.stdout.decode('utf-8') - except Exception as e: - sys.stdout.write("Failed to %s: %s\n" % (task_name, e)) - return None - -# collect source making configuration +# collect source package configuration +######################## +# config variables used: +# kernels: version of source kernel ebuild package to use [default='latest']; eg: 5.15.41, 5.15, latest +# distdir: name of distribution base directory [defalt='distribution'] +# package: name of kernel package configuration [mandatory] +# packages..pkg: name of source kernel ebuild package for this package configuration [mandatory] +# ebuild.cleanup: boolean flag indicating request to clean up temp files [default=True] +# ebuild.prefix: name of portage build base directory [default='/var/tmp/portage'] +# ebuild.tempdir: name of ktool build base directory [default='/var/tmp/prepare'] +# ebuild.format: name of package format configuration to use [mandatory] +# formats..extension: extension to use for this format configuration [mandatory] +# formats..archive: archive creation command to use for format configuration [mandatory]; variables: SRCDIR=source directory, DESTDIR=target directory, DESTFILENAME=target filename +# formats..extract: archive extraction command to use for format configuration [mandatory]; variables: SRCFILE=source file, DESTDIR=target directory +# context variables set up: +# arch: runtime architecture (x86, x64) +# pkg_name: name of kernel package configuration to use (copied from config variable 'package'); eg: mygentoo +# package_to_check: name and version of source kernel ebuild package to use (constructed from config variables 'packages..pkg' and 'kernels'); eg: sys-kernel/mygentoo-sources-5.15 +# build_prefix: name of portage build base directory (copied from config variable 'ebuild.prefix'); eg: /var/tmp/portage +# build_tempdir: name of ktool build base directory (copied from config variable 'ebuild.tempdir'); eg: /var/tmp/prepare +# distdir: name of distribution base directory (copied from config variable 'distdir'); eg: distribution +# build_clean: boolean flag indicating request to clean up temp files (copied from config variable 'ebuild.cleanup'); eg: True +# build_source_format: name of package format configuration to use (copied from config variable 'ebuild.format'); eg: tarxz +# build_source_extension: extension to use for package format (copied from config variable 'formats..extension'); eg: tar.xz +# build_source_archive: archive creation command to use for package format (copied from config variable 'formats..archive'); eg: tar cJfC +# build_source_extract: archive creation command to use for package format (copied from config variable 'formats..extract'); eg: tar xJfC +######################## def get_source_package_config(config, context): + arch = platform.machine() + if arch == 'i686': + arch = 'x86' + elif arch == 'x86_64': + arch = 'x64' + else: + sys.stdout.write("Unsupported cpu architecture: %s\n" % (arch)) + return 2 + context['arch'] = arch + config_pkg_name = config.get_config_param('package', mandatory=True) if config_pkg_name == None: return 3 + context['pkg_name'] = config_pkg_name config_pkg_id = 'packages.%s.pkg' % (config_pkg_name) package_name = config.get_config_param(config_pkg_id, mandatory=True) if package_name == None: @@ -42,9 +67,11 @@ def get_source_package_config(config, context): version_wanted = '' package_to_check = package_name if version_wanted == '' else '%s-%s*' % (package_name, version_wanted) build_prefix = config.get_config_param('ebuild.prefix', default=BUILD_PREFIX_DEFAULT) + build_tempdir = config.get_config_param('ebuild.tempdir', default=BUILD_TEMPDIR_DEFAULT) distdir = config.get_config_param('distdir', default=DISTDIR_DEFAULT) context['package_to_check'] = package_to_check context['build_prefix'] = build_prefix + context['build_tempdir'] = build_tempdir context['distdir'] = distdir build_source_format = config.get_config_param('ebuild.format', mandatory=True) @@ -63,11 +90,27 @@ def get_source_package_config(config, context): context['build_source_extension'] = build_source_extension context['build_source_archive'] = build_source_archive context['build_source_extract'] = build_source_extract + + for key in 'arch', 'pkg_name', 'build_prefix', 'build_tempdir', 'distdir', 'build_source_format', 'build_source_extension', 'build_source_archive', 'build_source_extract': + sys.stdout.write("%s: %s\n" % (key, context[key])) return 0 # determine source ebuild and version info +######################## +# context variables used: +# package_to_check (get_source_package_config) +# build_prefix (get_source_package_config) +# context variables set up: +# found_ebuild_filename: name of ebuild file found using 'equery which' based on context variable 'package_to_check'; eg: /var/db/repos/myoverlay/sys-kernel/mygentoo-sources/mygentoo-sources-5.15.41.ebuild +# found_package_group: package group parsed from ebuild file directory path ...///-.ebuild; eg: sys-kernel +# found_package_name: package name parsed from ebuild file directory path ...///-.ebuild; eg: mygentoo-sources +# found_version: package version parsed from ebuild file name ...///-.ebuild; eg: 5.15.41 +# found_package_fullname: package full atom identifier reconstructed from package group, package name, package version; eg: sys-kernel/mygentoo-sources-5.15.41 +# build_image_dir: name of image directory within portage package build directory (constructed from context variable 'build_prefix' and package full atom identifier); eg: /var/tmp/portage/sys-kernel/mygentoo-sources-5.15.41/image +# build_env_file: name of env file within portage package build directory (constructed from context variable 'build_prefix' and package full atom identifier); eg: /var/tmp/portage/sys-kernel/mygentoo-sources-5.15.41/temp/environment +######################## def get_source_ebuild_info(config, context): - result = run_command(config, "get ebuild file for package %s" % (context['package_to_check']), ['equery', 'which', context['package_to_check']]) + result = lib.run_command(config, "get ebuild file for package %s" % (context['package_to_check']), ['equery', 'which', context['package_to_check']]) if result == None: return 3 found_ebuild_filename = result.strip() @@ -87,16 +130,39 @@ def get_source_ebuild_info(config, context): sys.stdout.write("Package version found: %s\n" % (found_package_fullname)) context['found_ebuild_filename'] = found_ebuild_filename + context['found_package_group'] = found_package_group + context['found_package_name'] = found_package_name + context['found_version'] = found_version context['found_package_fullname'] = found_package_fullname context['build_image_dir'] = "%s/%s/image" % (context['build_prefix'], found_package_fullname) context['build_env_file'] = "%s/%s/temp/environment" % (context['build_prefix'], found_package_fullname) + + for key in 'found_ebuild_filename', 'found_package_fullname': + sys.stdout.write("%s: %s\n" % (key, context[key])) return 0 # extract build variables +######################## +# context variables used: +# distdir (get_source_package_config) +# build_source_extension (get_source_package_config) +# build_image_dir (get_source_ebuild_info) +# build_env_file (get_source_ebuild_info) +# context variables set up: +# kernel_version_string: build variable KV_FULL extracted; eg: 5.15.41-mygentoo +# kernel_version_major: build variable KV_MAJOR extracted; eg: 5 +# kernel_version_minor: build variable KV_MINOR extracted; eg: 15 +# kernel_version_patch: build variable KV_PATCH extracted; eg: 41 +# kernel_series: kernel series identifier (constructed from KV_MAJOR and KV_MINOR); eg: 5.15 +# kernel_series_distdir: distribution subdirectory for kernel series (constructed from config variable 'distdir' and KV_MAJOR and KV_MINOR); eg: distribution/5.15 +# build_kernel_dir: kernel source directory extracted into portage package build directory (constructed from context variables 'build_image_dir' and 'kernel_version_string'); eg: /var/tmp/portage/sys-kernel/mygentoo-sources-5.15.41/image//usr/src/linux-5.15.41-mygentoo +# source_archive_filename: kernel source archive filename (constructed from context variables 'kernel_version_string' and 'build_source_extension'); eg: 5.15.41-mygentoo.tar.xz +# source_archive_dist: kernel source archive subdirectory (constructed from context variables 'kernel_series_distdir' and 'source_archive_filename'); eg: distribution/5.15/5.15.41-mygentoo.tar.xz +######################## def extract_source_variables(config, context): vars = dict() for variable in ('KV_FULL', 'KV_MAJOR', 'KV_MINOR', 'KV_PATCH'): - result = run_command(config, "extract kernel version string from %s" % (context['build_env_file']), ['env', '-i', 'bash', '-c', "source %s && echo \"${%s}\"" % (context['build_env_file'], variable)]) + result = lib.run_command(config, "extract kernel version string from %s" % (context['build_env_file']), ['env', '-i', 'bash', '-c', "source %s && echo \"${%s}\"" % (context['build_env_file'], variable)]) if result == None: return "Could not extract kernel variable %s" % (variable) vars[variable] = result.strip() @@ -121,9 +187,12 @@ def extract_source_variables(config, context): context['source_archive_dist'] = "%s/%s" % (kernel_series_distdir, source_archive_filename) sys.stdout.write("Kernel version: %s.%s.%s full: %s\n" % (kernel_version_major, kernel_version_minor, kernel_version_patch, kernel_version_string)) + for key in 'kernel_version_string', 'kernel_series', 'kernel_series_distdir', 'source_archive_filename': + sys.stdout.write("%s: %s\n" % (key, context[key])) return "" # determine if source build is needed +######################## def check_source_build_needed(config, context): sys.stdout.write("Checking build environment file: %s\n" % (context['build_env_file'])) if not os.path.isfile(context['build_env_file']): @@ -131,7 +200,7 @@ def check_source_build_needed(config, context): sys.stdout.write("No kernel build environment file found but forced to rebuild source image anyway\n") return True sys.stdout.write("No kernel build environment file found\n") - result = run_command(config, "create kernel build environment file from ebuild %s" % (context['found_ebuild_filename']), ['ebuild', context['found_ebuild_filename'], 'clean', 'setup']) + result = lib.run_command(config, "create kernel build environment file from ebuild %s" % (context['found_ebuild_filename']), ['ebuild', context['found_ebuild_filename'], 'clean', 'setup']) if result == None: return True if not os.path.isfile(context['build_env_file']): @@ -159,6 +228,7 @@ def check_source_build_needed(config, context): return False # determine if making build archive is needed +######################## def check_build_archive_needed(config, context): if os.path.isfile(context['source_archive_dist']): if os.path.isdir(context['build_kernel_dir']): @@ -179,13 +249,14 @@ def check_build_archive_needed(config, context): return True # create archive for built source +######################## def make_build_source_archive(config, context): sys.stdout.write("Using build kernel directory: %s\n" % (context['build_kernel_dir'])) env = dict(os.environ) env['SRCDIR'] = context['build_kernel_dir'] env['DESTDIR'] = context['kernel_series_distdir'] env['DESTFILENAME'] = context['source_archive_filename'] - result = run_command(config, "create archive for kernel source %s" % (context['source_archive_dist']), ['bash', '-c', context['build_source_archive']], env) + result = lib.run_command(config, "create archive for kernel source %s" % (context['source_archive_dist']), ['bash', '-c', context['build_source_archive']], env) if result == None: return 3 if not os.path.isfile(context['source_archive_dist']): @@ -194,8 +265,9 @@ def make_build_source_archive(config, context): return 0 # build kernel source +######################## def build_source(config, context): - result = run_command(config, "create kernel source from ebuild %s" % (context['found_ebuild_filename']), ['ebuild', context['found_ebuild_filename'], 'clean', 'install']) + result = lib.run_command(config, "create kernel source from ebuild %s" % (context['found_ebuild_filename']), ['ebuild', context['found_ebuild_filename'], 'clean', 'install']) if result == None: return 3 if not os.path.isfile(context['build_env_file']): @@ -214,11 +286,13 @@ def build_source(config, context): return 0 # clear kernel source built +######################## def clear_source(config, context): - run_command(config, "clear kernel source for ebuild %s" % (context['found_ebuild_filename']), ['ebuild', context['found_ebuild_filename'], 'clean']) + lib.run_command(config, "clear kernel source for ebuild %s" % (context['found_ebuild_filename']), ['ebuild', context['found_ebuild_filename'], 'clean']) return 0 -# make source +# make source (mksrc command) +######################## def make_source(config): context = dict() rc = get_source_package_config(config, context) @@ -243,7 +317,12 @@ def make_source(config): return rc return None -# prepare source +# prepare source (prepare command) +######################## def prepare_source(config): - contetx = dict() + context = dict() + rc = get_source_package_config(config, context) + if rc: + return rc + return None