Refactored libs & add logging
Improved docs
This commit is contained in:
parent
207f637f37
commit
131b7e1d5e
|
|
@ -9,7 +9,7 @@
|
|||
- emerge unmerge
|
||||
|
||||
```bash
|
||||
ktool mksrc
|
||||
ktool mksrc [-s kernels=4.19]
|
||||
```
|
||||
|
||||
### Prepare kernel source
|
||||
|
|
@ -21,7 +21,7 @@ ktool mksrc
|
|||
- store prepared kernel source
|
||||
|
||||
```bash
|
||||
ktool prepare
|
||||
ktool prepare [-s kernels=4.19]
|
||||
```
|
||||
|
||||
### Build kernel source package
|
||||
|
|
@ -74,3 +74,5 @@ ktool mkucodes
|
|||
```bash
|
||||
ktool install
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
|
|
|||
|
|
@ -18,6 +18,8 @@ ebuild:
|
|||
## prefix: /var/tmp/portage
|
||||
# format to use to store source, must exist under 'formats'
|
||||
format: tarxz
|
||||
# source preparation temp directory
|
||||
tempdir: /var/tmp/prepare
|
||||
|
||||
# format descriptions
|
||||
# extension: extension to use for this format
|
||||
|
|
|
|||
2
ktool
2
ktool
|
|
@ -6,7 +6,7 @@ import sys
|
|||
|
||||
import tool
|
||||
import progver
|
||||
from configuration import Config
|
||||
from parseconfig import Config
|
||||
|
||||
config = Config(__file__, progver.VERSION, [
|
||||
['mksrc', 'Make kernel source'],
|
||||
|
|
|
|||
21
lib.py
Normal file
21
lib.py
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
# -*- coding: UTF-8 -*-
|
||||
|
||||
import sys
|
||||
import subprocess
|
||||
from log import *
|
||||
|
||||
# run command and return stdout or None in case of error
|
||||
########################
|
||||
def run_command(config, task_name, argv, env = None):
|
||||
log(LOG_INFO, "Running command '%s' to %s" % (' '.join(argv), task_name))
|
||||
try:
|
||||
result = subprocess.run(argv, stdout=subprocess.PIPE, env=env)
|
||||
if config.verbose:
|
||||
log(LOG_INFO, "Command output: %s" % (result.stdout.decode('utf-8')))
|
||||
if result.returncode != 0:
|
||||
log(LOG_ERROR, "Failed to %s: exit code %d" % (task_name, result.returncode))
|
||||
return None
|
||||
return result.stdout.decode('utf-8')
|
||||
except Exception as e:
|
||||
log(LOG_ERROR, "Failed to %s: %s" % (task_name, e))
|
||||
return None
|
||||
25
log.py
Normal file
25
log.py
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
# -*- coding: UTF-8 -*-
|
||||
|
||||
import sys
|
||||
|
||||
LOG_FATAL = 0
|
||||
LOG_ERROR = 1
|
||||
LOG_WARNING = 2
|
||||
LOG_INFO = 3
|
||||
LOG_DEBUG = 4
|
||||
|
||||
_log_level = LOG_INFO
|
||||
|
||||
# set log level
|
||||
########################
|
||||
def set_log_level(level):
|
||||
_log_level = level
|
||||
|
||||
# log messages
|
||||
########################
|
||||
def log(level, message):
|
||||
if level <= _log_level:
|
||||
if level >= LOG_ERROR:
|
||||
sys.stdout.write("%s\n" % (message))
|
||||
else:
|
||||
sys.stderr.write("%s\n" % (message))
|
||||
|
|
@ -11,23 +11,23 @@ except ImportError:
|
|||
from yaml import SafeLoader
|
||||
from pathlib import Path
|
||||
|
||||
def merge_config(src, dest):
|
||||
def _merge_config(src, dest):
|
||||
for key, value in src.items():
|
||||
if isinstance(value, dict):
|
||||
node = dest.setdefault(key, {})
|
||||
merge_config(value, node)
|
||||
_merge_config(value, node)
|
||||
else:
|
||||
dest[key] = value
|
||||
return dest
|
||||
|
||||
def parse_config_set(params, a):
|
||||
def _parse_config_set(params, a):
|
||||
a = a.strip()
|
||||
if a.startswith('['):
|
||||
return "config parameters are not of list type"
|
||||
if a.startswith('{'):
|
||||
try:
|
||||
value = json.loads(a)
|
||||
merge_config(value, params)
|
||||
_merge_config(value, params)
|
||||
return None
|
||||
except json.JSONDecodeError as e:
|
||||
return "failed to decode JSON struct: %s" % (e)
|
||||
|
|
@ -120,7 +120,7 @@ Options:
|
|||
self.usage(2)
|
||||
self.config = a
|
||||
elif o == '-s' or o == "--set":
|
||||
error = parse_config_set(self.params, a)
|
||||
error = _parse_config_set(self.params, a)
|
||||
if error:
|
||||
sys.stderr.write("Error: option %s argument is invalid: %s\n" % (o, error))
|
||||
self.usage(2)
|
||||
|
|
@ -149,7 +149,7 @@ Options:
|
|||
sys.stderr.write("Missing command\n")
|
||||
self.usage(2)
|
||||
|
||||
merge_config(self.params, self.config_params)
|
||||
_merge_config(self.params, self.config_params)
|
||||
if self.verbose:
|
||||
sys.stdout.write("Config parameters:\n")
|
||||
for k, v in self.config_params.items():
|
||||
|
|
@ -158,7 +158,7 @@ Options:
|
|||
def parse_file(self, filepath):
|
||||
try:
|
||||
obj = load(open(filepath, 'r'), SafeLoader)
|
||||
merge_config(obj, self.config_params)
|
||||
_merge_config(obj, self.config_params)
|
||||
except YAMLError as exc:
|
||||
sys.stdout.write("Error: reading configuration file failed: %s" % (exc))
|
||||
|
||||
|
|
@ -2,5 +2,4 @@
|
|||
|
||||
import sys
|
||||
|
||||
this = sys.modules[__name__]
|
||||
this.VERSION = "0.01"
|
||||
VERSION = "0.01"
|
||||
|
|
|
|||
131
tool.py
131
tool.py
|
|
@ -3,32 +3,57 @@
|
|||
import os
|
||||
import re
|
||||
import sys
|
||||
import subprocess
|
||||
import platform
|
||||
import lib
|
||||
from log import *
|
||||
|
||||
DISTDIR_DEFAULT = 'distribution'
|
||||
BUILD_PREFIX_DEFAULT = '/var/tmp/portage'
|
||||
BUILD_TEMPDIR_DEFAULT = '/var/tmp/prepare'
|
||||
PARSE_EBUILD_FILENAME = re.compile("^.*/([^/]+)/([^/]+)/([^/]+)\.ebuild$")
|
||||
|
||||
# run command
|
||||
def run_command(config, task_name, argv, env = None):
|
||||
sys.stdout.write("Running command '%s' to %s\n" % (' '.join(argv), task_name))
|
||||
try:
|
||||
result = subprocess.run(argv, stdout=subprocess.PIPE, env=env)
|
||||
if config.verbose:
|
||||
sys.stdout.write("Command output: %s\n" % (result.stdout.decode('utf-8')))
|
||||
if result.returncode != 0:
|
||||
sys.stdout.write("Failed to %s: exit code %d\n" % (task_name, result.returncode))
|
||||
return None
|
||||
return result.stdout.decode('utf-8')
|
||||
except Exception as e:
|
||||
sys.stdout.write("Failed to %s: %s\n" % (task_name, e))
|
||||
return None
|
||||
|
||||
# collect source making configuration
|
||||
# collect source package configuration
|
||||
########################
|
||||
# config variables used:
|
||||
# kernels: version of source kernel ebuild package to use [default='latest']; eg: 5.15.41, 5.15, latest
|
||||
# distdir: name of distribution base directory [defalt='distribution']
|
||||
# package: name of kernel package configuration [mandatory]
|
||||
# packages.<package>.pkg: name of source kernel ebuild package for this package configuration [mandatory]
|
||||
# ebuild.cleanup: boolean flag indicating request to clean up temp files [default=True]
|
||||
# ebuild.prefix: name of portage build base directory [default='/var/tmp/portage']
|
||||
# ebuild.tempdir: name of ktool build base directory [default='/var/tmp/prepare']
|
||||
# ebuild.format: name of package format configuration to use [mandatory]
|
||||
# formats.<format>.extension: extension to use for this format configuration [mandatory]
|
||||
# formats.<format>.archive: archive creation command to use for format configuration [mandatory]; variables: SRCDIR=source directory, DESTDIR=target directory, DESTFILENAME=target filename
|
||||
# formats.<format>.extract: archive extraction command to use for format configuration [mandatory]; variables: SRCFILE=source file, DESTDIR=target directory
|
||||
# context variables set up:
|
||||
# arch: runtime architecture (x86, x64)
|
||||
# pkg_name: name of kernel package configuration to use (copied from config variable 'package'); eg: mygentoo
|
||||
# package_to_check: name and version of source kernel ebuild package to use (constructed from config variables 'packages.<package>.pkg' and 'kernels'); eg: sys-kernel/mygentoo-sources-5.15
|
||||
# build_prefix: name of portage build base directory (copied from config variable 'ebuild.prefix'); eg: /var/tmp/portage
|
||||
# build_tempdir: name of ktool build base directory (copied from config variable 'ebuild.tempdir'); eg: /var/tmp/prepare
|
||||
# distdir: name of distribution base directory (copied from config variable 'distdir'); eg: distribution
|
||||
# build_clean: boolean flag indicating request to clean up temp files (copied from config variable 'ebuild.cleanup'); eg: True
|
||||
# build_source_format: name of package format configuration to use (copied from config variable 'ebuild.format'); eg: tarxz
|
||||
# build_source_extension: extension to use for package format (copied from config variable 'formats.<format>.extension'); eg: tar.xz
|
||||
# build_source_archive: archive creation command to use for package format (copied from config variable 'formats.<format>.archive'); eg: tar cJfC
|
||||
# build_source_extract: archive creation command to use for package format (copied from config variable 'formats.<format>.extract'); eg: tar xJfC
|
||||
########################
|
||||
def get_source_package_config(config, context):
|
||||
arch = platform.machine()
|
||||
if arch == 'i686':
|
||||
arch = 'x86'
|
||||
elif arch == 'x86_64':
|
||||
arch = 'x64'
|
||||
else:
|
||||
sys.stdout.write("Unsupported cpu architecture: %s\n" % (arch))
|
||||
return 2
|
||||
context['arch'] = arch
|
||||
|
||||
config_pkg_name = config.get_config_param('package', mandatory=True)
|
||||
if config_pkg_name == None:
|
||||
return 3
|
||||
context['pkg_name'] = config_pkg_name
|
||||
config_pkg_id = 'packages.%s.pkg' % (config_pkg_name)
|
||||
package_name = config.get_config_param(config_pkg_id, mandatory=True)
|
||||
if package_name == None:
|
||||
|
|
@ -42,9 +67,11 @@ def get_source_package_config(config, context):
|
|||
version_wanted = ''
|
||||
package_to_check = package_name if version_wanted == '' else '%s-%s*' % (package_name, version_wanted)
|
||||
build_prefix = config.get_config_param('ebuild.prefix', default=BUILD_PREFIX_DEFAULT)
|
||||
build_tempdir = config.get_config_param('ebuild.tempdir', default=BUILD_TEMPDIR_DEFAULT)
|
||||
distdir = config.get_config_param('distdir', default=DISTDIR_DEFAULT)
|
||||
context['package_to_check'] = package_to_check
|
||||
context['build_prefix'] = build_prefix
|
||||
context['build_tempdir'] = build_tempdir
|
||||
context['distdir'] = distdir
|
||||
|
||||
build_source_format = config.get_config_param('ebuild.format', mandatory=True)
|
||||
|
|
@ -63,11 +90,27 @@ def get_source_package_config(config, context):
|
|||
context['build_source_extension'] = build_source_extension
|
||||
context['build_source_archive'] = build_source_archive
|
||||
context['build_source_extract'] = build_source_extract
|
||||
|
||||
for key in 'arch', 'pkg_name', 'build_prefix', 'build_tempdir', 'distdir', 'build_source_format', 'build_source_extension', 'build_source_archive', 'build_source_extract':
|
||||
sys.stdout.write("%s: %s\n" % (key, context[key]))
|
||||
return 0
|
||||
|
||||
# determine source ebuild and version info
|
||||
########################
|
||||
# context variables used:
|
||||
# package_to_check (get_source_package_config)
|
||||
# build_prefix (get_source_package_config)
|
||||
# context variables set up:
|
||||
# found_ebuild_filename: name of ebuild file found using 'equery which' based on context variable 'package_to_check'; eg: /var/db/repos/myoverlay/sys-kernel/mygentoo-sources/mygentoo-sources-5.15.41.ebuild
|
||||
# found_package_group: package group parsed from ebuild file directory path .../<pkggroup>/<pkgname>/<pkgname>-<version>.ebuild; eg: sys-kernel
|
||||
# found_package_name: package name parsed from ebuild file directory path .../<pkggroup>/<pkgname>/<pkgname>-<version>.ebuild; eg: mygentoo-sources
|
||||
# found_version: package version parsed from ebuild file name .../<pkggroup>/<pkgname>/<pkgname>-<version>.ebuild; eg: 5.15.41
|
||||
# found_package_fullname: package full atom identifier reconstructed from package group, package name, package version; eg: sys-kernel/mygentoo-sources-5.15.41
|
||||
# build_image_dir: name of image directory within portage package build directory (constructed from context variable 'build_prefix' and package full atom identifier); eg: /var/tmp/portage/sys-kernel/mygentoo-sources-5.15.41/image
|
||||
# build_env_file: name of env file within portage package build directory (constructed from context variable 'build_prefix' and package full atom identifier); eg: /var/tmp/portage/sys-kernel/mygentoo-sources-5.15.41/temp/environment
|
||||
########################
|
||||
def get_source_ebuild_info(config, context):
|
||||
result = run_command(config, "get ebuild file for package %s" % (context['package_to_check']), ['equery', 'which', context['package_to_check']])
|
||||
result = lib.run_command(config, "get ebuild file for package %s" % (context['package_to_check']), ['equery', 'which', context['package_to_check']])
|
||||
if result == None:
|
||||
return 3
|
||||
found_ebuild_filename = result.strip()
|
||||
|
|
@ -87,16 +130,39 @@ def get_source_ebuild_info(config, context):
|
|||
sys.stdout.write("Package version found: %s\n" % (found_package_fullname))
|
||||
|
||||
context['found_ebuild_filename'] = found_ebuild_filename
|
||||
context['found_package_group'] = found_package_group
|
||||
context['found_package_name'] = found_package_name
|
||||
context['found_version'] = found_version
|
||||
context['found_package_fullname'] = found_package_fullname
|
||||
context['build_image_dir'] = "%s/%s/image" % (context['build_prefix'], found_package_fullname)
|
||||
context['build_env_file'] = "%s/%s/temp/environment" % (context['build_prefix'], found_package_fullname)
|
||||
|
||||
for key in 'found_ebuild_filename', 'found_package_fullname':
|
||||
sys.stdout.write("%s: %s\n" % (key, context[key]))
|
||||
return 0
|
||||
|
||||
# extract build variables
|
||||
########################
|
||||
# context variables used:
|
||||
# distdir (get_source_package_config)
|
||||
# build_source_extension (get_source_package_config)
|
||||
# build_image_dir (get_source_ebuild_info)
|
||||
# build_env_file (get_source_ebuild_info)
|
||||
# context variables set up:
|
||||
# kernel_version_string: build variable KV_FULL extracted; eg: 5.15.41-mygentoo
|
||||
# kernel_version_major: build variable KV_MAJOR extracted; eg: 5
|
||||
# kernel_version_minor: build variable KV_MINOR extracted; eg: 15
|
||||
# kernel_version_patch: build variable KV_PATCH extracted; eg: 41
|
||||
# kernel_series: kernel series identifier (constructed from KV_MAJOR and KV_MINOR); eg: 5.15
|
||||
# kernel_series_distdir: distribution subdirectory for kernel series (constructed from config variable 'distdir' and KV_MAJOR and KV_MINOR); eg: distribution/5.15
|
||||
# build_kernel_dir: kernel source directory extracted into portage package build directory (constructed from context variables 'build_image_dir' and 'kernel_version_string'); eg: /var/tmp/portage/sys-kernel/mygentoo-sources-5.15.41/image//usr/src/linux-5.15.41-mygentoo
|
||||
# source_archive_filename: kernel source archive filename (constructed from context variables 'kernel_version_string' and 'build_source_extension'); eg: 5.15.41-mygentoo.tar.xz
|
||||
# source_archive_dist: kernel source archive subdirectory (constructed from context variables 'kernel_series_distdir' and 'source_archive_filename'); eg: distribution/5.15/5.15.41-mygentoo.tar.xz
|
||||
########################
|
||||
def extract_source_variables(config, context):
|
||||
vars = dict()
|
||||
for variable in ('KV_FULL', 'KV_MAJOR', 'KV_MINOR', 'KV_PATCH'):
|
||||
result = run_command(config, "extract kernel version string from %s" % (context['build_env_file']), ['env', '-i', 'bash', '-c', "source %s && echo \"${%s}\"" % (context['build_env_file'], variable)])
|
||||
result = lib.run_command(config, "extract kernel version string from %s" % (context['build_env_file']), ['env', '-i', 'bash', '-c', "source %s && echo \"${%s}\"" % (context['build_env_file'], variable)])
|
||||
if result == None:
|
||||
return "Could not extract kernel variable %s" % (variable)
|
||||
vars[variable] = result.strip()
|
||||
|
|
@ -121,9 +187,12 @@ def extract_source_variables(config, context):
|
|||
context['source_archive_dist'] = "%s/%s" % (kernel_series_distdir, source_archive_filename)
|
||||
sys.stdout.write("Kernel version: %s.%s.%s full: %s\n" % (kernel_version_major, kernel_version_minor, kernel_version_patch, kernel_version_string))
|
||||
|
||||
for key in 'kernel_version_string', 'kernel_series', 'kernel_series_distdir', 'source_archive_filename':
|
||||
sys.stdout.write("%s: %s\n" % (key, context[key]))
|
||||
return ""
|
||||
|
||||
# determine if source build is needed
|
||||
########################
|
||||
def check_source_build_needed(config, context):
|
||||
sys.stdout.write("Checking build environment file: %s\n" % (context['build_env_file']))
|
||||
if not os.path.isfile(context['build_env_file']):
|
||||
|
|
@ -131,7 +200,7 @@ def check_source_build_needed(config, context):
|
|||
sys.stdout.write("No kernel build environment file found but forced to rebuild source image anyway\n")
|
||||
return True
|
||||
sys.stdout.write("No kernel build environment file found\n")
|
||||
result = run_command(config, "create kernel build environment file from ebuild %s" % (context['found_ebuild_filename']), ['ebuild', context['found_ebuild_filename'], 'clean', 'setup'])
|
||||
result = lib.run_command(config, "create kernel build environment file from ebuild %s" % (context['found_ebuild_filename']), ['ebuild', context['found_ebuild_filename'], 'clean', 'setup'])
|
||||
if result == None:
|
||||
return True
|
||||
if not os.path.isfile(context['build_env_file']):
|
||||
|
|
@ -159,6 +228,7 @@ def check_source_build_needed(config, context):
|
|||
return False
|
||||
|
||||
# determine if making build archive is needed
|
||||
########################
|
||||
def check_build_archive_needed(config, context):
|
||||
if os.path.isfile(context['source_archive_dist']):
|
||||
if os.path.isdir(context['build_kernel_dir']):
|
||||
|
|
@ -179,13 +249,14 @@ def check_build_archive_needed(config, context):
|
|||
return True
|
||||
|
||||
# create archive for built source
|
||||
########################
|
||||
def make_build_source_archive(config, context):
|
||||
sys.stdout.write("Using build kernel directory: %s\n" % (context['build_kernel_dir']))
|
||||
env = dict(os.environ)
|
||||
env['SRCDIR'] = context['build_kernel_dir']
|
||||
env['DESTDIR'] = context['kernel_series_distdir']
|
||||
env['DESTFILENAME'] = context['source_archive_filename']
|
||||
result = run_command(config, "create archive for kernel source %s" % (context['source_archive_dist']), ['bash', '-c', context['build_source_archive']], env)
|
||||
result = lib.run_command(config, "create archive for kernel source %s" % (context['source_archive_dist']), ['bash', '-c', context['build_source_archive']], env)
|
||||
if result == None:
|
||||
return 3
|
||||
if not os.path.isfile(context['source_archive_dist']):
|
||||
|
|
@ -194,8 +265,9 @@ def make_build_source_archive(config, context):
|
|||
return 0
|
||||
|
||||
# build kernel source
|
||||
########################
|
||||
def build_source(config, context):
|
||||
result = run_command(config, "create kernel source from ebuild %s" % (context['found_ebuild_filename']), ['ebuild', context['found_ebuild_filename'], 'clean', 'install'])
|
||||
result = lib.run_command(config, "create kernel source from ebuild %s" % (context['found_ebuild_filename']), ['ebuild', context['found_ebuild_filename'], 'clean', 'install'])
|
||||
if result == None:
|
||||
return 3
|
||||
if not os.path.isfile(context['build_env_file']):
|
||||
|
|
@ -214,11 +286,13 @@ def build_source(config, context):
|
|||
return 0
|
||||
|
||||
# clear kernel source built
|
||||
########################
|
||||
def clear_source(config, context):
|
||||
run_command(config, "clear kernel source for ebuild %s" % (context['found_ebuild_filename']), ['ebuild', context['found_ebuild_filename'], 'clean'])
|
||||
lib.run_command(config, "clear kernel source for ebuild %s" % (context['found_ebuild_filename']), ['ebuild', context['found_ebuild_filename'], 'clean'])
|
||||
return 0
|
||||
|
||||
# make source
|
||||
# make source (mksrc command)
|
||||
########################
|
||||
def make_source(config):
|
||||
context = dict()
|
||||
rc = get_source_package_config(config, context)
|
||||
|
|
@ -243,7 +317,12 @@ def make_source(config):
|
|||
return rc
|
||||
return None
|
||||
|
||||
# prepare source
|
||||
# prepare source (prepare command)
|
||||
########################
|
||||
def prepare_source(config):
|
||||
contetx = dict()
|
||||
context = dict()
|
||||
rc = get_source_package_config(config, context)
|
||||
if rc:
|
||||
return rc
|
||||
|
||||
return None
|
||||
|
|
|
|||
Loading…
Reference in a new issue