Refactored libs & add logging

Improved docs

README.md

@@ -9,7 +9,7 @@
 - emerge unmerge
 
 ```bash
-ktool mksrc
+ktool mksrc [-s kernels=4.19]
 ```
 
 ### Prepare kernel source
@@ -21,7 +21,7 @@ ktool mksrc
 - store prepared kernel source
 
 ```bash
-ktool prepare
+ktool prepare [-s kernels=4.19]
 ```
 
 ### Build kernel source package
@@ -74,3 +74,5 @@ ktool mkucodes
 ```bash
 ktool install
 ```
+
+## Configuration

config/00defaults.yaml

@@ -18,6 +18,8 @@ ebuild:
 ##  prefix: /var/tmp/portage
 # format to use to store source, must exist under 'formats'
   format: tarxz
+# source preparation temp directory
+  tempdir: /var/tmp/prepare
 
 # format descriptions
 #   extension: extension to use for this format

ktool

@@ -6,7 +6,7 @@ import sys
 
 import tool
 import progver
-from configuration import Config
+from parseconfig import Config
 
 config = Config(__file__, progver.VERSION, [
   ['mksrc', 'Make kernel source'],

lib.py

@@ -0,0 +1,21 @@
+# -*- coding: UTF-8 -*-
+
+import sys
+import subprocess
+from log import *
+
+# run command and return stdout or None in case of error
+########################
+def run_command(config, task_name, argv, env = None):
+  log(LOG_INFO, "Running command '%s' to %s" % (' '.join(argv), task_name))
+  try:
+    result = subprocess.run(argv, stdout=subprocess.PIPE, env=env)
+    if config.verbose:
+      log(LOG_INFO, "Command output: %s" % (result.stdout.decode('utf-8')))
+    if result.returncode != 0:
+      log(LOG_ERROR, "Failed to %s: exit code %d" % (task_name, result.returncode))
+      return None
+    return result.stdout.decode('utf-8')
+  except Exception as e:
+    log(LOG_ERROR, "Failed to %s: %s" % (task_name, e))
+    return None

log.py

@@ -0,0 +1,25 @@
+# -*- coding: UTF-8 -*-
+
+import sys
+
+LOG_FATAL = 0
+LOG_ERROR = 1
+LOG_WARNING = 2
+LOG_INFO = 3
+LOG_DEBUG = 4
+
+_log_level = LOG_INFO
+
+# set log level
+########################
+def set_log_level(level):
+  _log_level = level
+
+# log messages
+########################
+def log(level, message):
+  if level <= _log_level:
+    if level >= LOG_ERROR:
+      sys.stdout.write("%s\n" % (message))
+    else:
+      sys.stderr.write("%s\n" % (message))

parseconfig.py

@@ -11,23 +11,23 @@ except ImportError:
   from yaml import SafeLoader
 from pathlib import Path
 
-def merge_config(src, dest):
+def _merge_config(src, dest):
   for key, value in src.items():
     if isinstance(value, dict):
       node = dest.setdefault(key, {})
-      merge_config(value, node)
+      _merge_config(value, node)
     else:
       dest[key] = value
   return dest
 
-def parse_config_set(params, a):
+def _parse_config_set(params, a):
   a = a.strip()
   if a.startswith('['):
     return "config parameters are not of list type"
   if a.startswith('{'):
     try:
       value = json.loads(a)
-      merge_config(value, params)
+      _merge_config(value, params)
       return None
     except json.JSONDecodeError as e:
       return "failed to decode JSON struct: %s" % (e)
@@ -120,7 +120,7 @@ Options:
           self.usage(2)
         self.config = a
       elif o == '-s' or o == "--set":
-        error = parse_config_set(self.params, a)
+        error = _parse_config_set(self.params, a)
         if error:
           sys.stderr.write("Error: option %s argument is invalid: %s\n" % (o, error))
           self.usage(2)
@@ -149,7 +149,7 @@ Options:
       sys.stderr.write("Missing command\n")
       self.usage(2)
 
-    merge_config(self.params, self.config_params)
+    _merge_config(self.params, self.config_params)
     if self.verbose:
       sys.stdout.write("Config parameters:\n")
       for k, v in self.config_params.items():
@@ -158,7 +158,7 @@ Options:
   def parse_file(self, filepath):
     try:
       obj = load(open(filepath, 'r'), SafeLoader)
-      merge_config(obj, self.config_params)
+      _merge_config(obj, self.config_params)
     except YAMLError as exc:
       sys.stdout.write("Error: reading configuration file failed: %s" % (exc))
 

progver.py

@@ -2,5 +2,4 @@
 
 import sys
 
-this = sys.modules[__name__]
+VERSION = "0.01"
-this.VERSION = "0.01"

tool.py

@@ -3,32 +3,57 @@
 import os
 import re
 import sys
-import subprocess
+import platform
+import lib
+from log import *
 
 DISTDIR_DEFAULT = 'distribution'
 BUILD_PREFIX_DEFAULT = '/var/tmp/portage'
+BUILD_TEMPDIR_DEFAULT = '/var/tmp/prepare'
 PARSE_EBUILD_FILENAME = re.compile("^.*/([^/]+)/([^/]+)/([^/]+)\.ebuild$")
 
-# run command
+# collect source package configuration
-def run_command(config, task_name, argv, env = None):
+########################
-  sys.stdout.write("Running command '%s' to %s\n" % (' '.join(argv), task_name))
+# config variables used:
-  try:
+#   kernels: version of source kernel ebuild package to use [default='latest']; eg: 5.15.41, 5.15, latest
-    result = subprocess.run(argv, stdout=subprocess.PIPE, env=env)
+#   distdir: name of distribution base directory [defalt='distribution']
-    if config.verbose:
+#   package: name of kernel package configuration [mandatory]
-      sys.stdout.write("Command output: %s\n" % (result.stdout.decode('utf-8')))
+#   packages.<package>.pkg: name of source kernel ebuild package for this package configuration [mandatory]
-    if result.returncode != 0:
+#   ebuild.cleanup: boolean flag indicating request to clean up temp files [default=True]
-      sys.stdout.write("Failed to %s: exit code %d\n" % (task_name, result.returncode))
+#   ebuild.prefix: name of portage build base directory [default='/var/tmp/portage']
-      return None
+#   ebuild.tempdir: name of ktool build base directory [default='/var/tmp/prepare']
-    return result.stdout.decode('utf-8')
+#   ebuild.format: name of package format configuration to use [mandatory]
-  except Exception as e:
+#   formats.<format>.extension: extension to use for this format configuration [mandatory]
-    sys.stdout.write("Failed to %s: %s\n" % (task_name, e))
+#   formats.<format>.archive: archive creation command to use for format configuration [mandatory]; variables: SRCDIR=source directory, DESTDIR=target directory, DESTFILENAME=target filename
-    return None
+#   formats.<format>.extract: archive extraction command to use for format configuration [mandatory]; variables: SRCFILE=source file, DESTDIR=target directory
-
+# context variables set up:
-# collect source making configuration
+#   arch: runtime architecture (x86, x64)
+#   pkg_name: name of kernel package configuration to use (copied from config variable 'package'); eg: mygentoo
+#   package_to_check: name and version of source kernel ebuild package to use (constructed from config variables 'packages.<package>.pkg' and 'kernels'); eg: sys-kernel/mygentoo-sources-5.15
+#   build_prefix: name of portage build base directory (copied from config variable 'ebuild.prefix'); eg: /var/tmp/portage
+#   build_tempdir: name of ktool build base directory (copied from config variable 'ebuild.tempdir'); eg: /var/tmp/prepare
+#   distdir: name of distribution base directory (copied from config variable 'distdir'); eg: distribution
+#   build_clean: boolean flag indicating request to clean up temp files (copied from config variable 'ebuild.cleanup'); eg: True
+#   build_source_format: name of package format configuration to use (copied from config variable 'ebuild.format'); eg: tarxz
+#   build_source_extension: extension to use for package format (copied from config variable 'formats.<format>.extension'); eg: tar.xz
+#   build_source_archive: archive creation command to use for package format (copied from config variable 'formats.<format>.archive'); eg: tar cJfC
+#   build_source_extract: archive creation command to use for package format (copied from config variable 'formats.<format>.extract'); eg: tar xJfC
+########################
 def get_source_package_config(config, context):
+  arch = platform.machine()
+  if arch == 'i686':
+    arch = 'x86'
+  elif arch == 'x86_64':
+    arch = 'x64'
+  else:
+    sys.stdout.write("Unsupported cpu architecture: %s\n" % (arch))
+    return 2
+  context['arch'] = arch
+
   config_pkg_name = config.get_config_param('package', mandatory=True)
   if config_pkg_name == None:
     return 3
+  context['pkg_name'] = config_pkg_name
   config_pkg_id = 'packages.%s.pkg' % (config_pkg_name)
   package_name = config.get_config_param(config_pkg_id, mandatory=True)
   if package_name == None:
@@ -42,9 +67,11 @@ def get_source_package_config(config, context):
     version_wanted = ''
   package_to_check = package_name if version_wanted == '' else '%s-%s*' % (package_name, version_wanted)
   build_prefix = config.get_config_param('ebuild.prefix', default=BUILD_PREFIX_DEFAULT)
+  build_tempdir = config.get_config_param('ebuild.tempdir', default=BUILD_TEMPDIR_DEFAULT)
   distdir = config.get_config_param('distdir', default=DISTDIR_DEFAULT)
   context['package_to_check'] = package_to_check
   context['build_prefix'] = build_prefix
+  context['build_tempdir'] = build_tempdir
   context['distdir'] = distdir
 
   build_source_format = config.get_config_param('ebuild.format', mandatory=True)
@@ -63,11 +90,27 @@ def get_source_package_config(config, context):
   context['build_source_extension'] = build_source_extension
   context['build_source_archive'] = build_source_archive
   context['build_source_extract'] = build_source_extract
+
+  for key in 'arch', 'pkg_name', 'build_prefix', 'build_tempdir', 'distdir', 'build_source_format', 'build_source_extension', 'build_source_archive', 'build_source_extract':
+    sys.stdout.write("%s: %s\n" % (key, context[key]))
   return 0
 
 # determine source ebuild and version info
+########################
+# context variables used:
+#   package_to_check (get_source_package_config)
+#   build_prefix (get_source_package_config)
+# context variables set up:
+#   found_ebuild_filename: name of ebuild file found using 'equery which' based on context variable 'package_to_check'; eg: /var/db/repos/myoverlay/sys-kernel/mygentoo-sources/mygentoo-sources-5.15.41.ebuild
+#   found_package_group: package group parsed from ebuild file directory path .../<pkggroup>/<pkgname>/<pkgname>-<version>.ebuild; eg: sys-kernel
+#   found_package_name: package name parsed from ebuild file directory path .../<pkggroup>/<pkgname>/<pkgname>-<version>.ebuild; eg: mygentoo-sources
+#   found_version: package version parsed from ebuild file name .../<pkggroup>/<pkgname>/<pkgname>-<version>.ebuild; eg: 5.15.41
+#   found_package_fullname: package full atom identifier reconstructed from package group, package name, package version; eg: sys-kernel/mygentoo-sources-5.15.41
+#   build_image_dir: name of image directory within portage package build directory (constructed from context variable 'build_prefix' and package full atom identifier); eg: /var/tmp/portage/sys-kernel/mygentoo-sources-5.15.41/image
+#   build_env_file: name of env file within portage package build directory (constructed from context variable 'build_prefix' and package full atom identifier); eg: /var/tmp/portage/sys-kernel/mygentoo-sources-5.15.41/temp/environment
+########################
 def get_source_ebuild_info(config, context):
-  result = run_command(config, "get ebuild file for package %s" % (context['package_to_check']), ['equery', 'which', context['package_to_check']])
+  result = lib.run_command(config, "get ebuild file for package %s" % (context['package_to_check']), ['equery', 'which', context['package_to_check']])
   if result == None:
     return 3
   found_ebuild_filename = result.strip()
@@ -87,16 +130,39 @@ def get_source_ebuild_info(config, context):
   sys.stdout.write("Package version found: %s\n" % (found_package_fullname))
 
   context['found_ebuild_filename'] = found_ebuild_filename
+  context['found_package_group'] = found_package_group
+  context['found_package_name'] = found_package_name
+  context['found_version'] = found_version
   context['found_package_fullname'] = found_package_fullname
   context['build_image_dir'] = "%s/%s/image" % (context['build_prefix'], found_package_fullname)
   context['build_env_file'] = "%s/%s/temp/environment" % (context['build_prefix'], found_package_fullname)
+
+  for key in 'found_ebuild_filename', 'found_package_fullname':
+    sys.stdout.write("%s: %s\n" % (key, context[key]))
   return 0
 
 # extract build variables
+########################
+# context variables used:
+#   distdir (get_source_package_config)
+#   build_source_extension (get_source_package_config)
+#   build_image_dir (get_source_ebuild_info)
+#   build_env_file (get_source_ebuild_info)
+# context variables set up:
+#   kernel_version_string: build variable KV_FULL extracted; eg: 5.15.41-mygentoo
+#   kernel_version_major: build variable KV_MAJOR extracted; eg: 5
+#   kernel_version_minor: build variable KV_MINOR extracted; eg: 15
+#   kernel_version_patch: build variable KV_PATCH extracted; eg: 41
+#   kernel_series: kernel series identifier (constructed from KV_MAJOR and KV_MINOR); eg: 5.15
+#   kernel_series_distdir: distribution subdirectory for kernel series (constructed from config variable 'distdir' and KV_MAJOR and KV_MINOR); eg: distribution/5.15
+#   build_kernel_dir: kernel source directory extracted into portage package build directory (constructed from context variables 'build_image_dir' and 'kernel_version_string'); eg: /var/tmp/portage/sys-kernel/mygentoo-sources-5.15.41/image//usr/src/linux-5.15.41-mygentoo
+#   source_archive_filename: kernel source archive filename (constructed from context variables 'kernel_version_string' and 'build_source_extension'); eg: 5.15.41-mygentoo.tar.xz
+#   source_archive_dist: kernel source archive subdirectory (constructed from context variables 'kernel_series_distdir' and 'source_archive_filename'); eg: distribution/5.15/5.15.41-mygentoo.tar.xz
+########################
 def extract_source_variables(config, context):
   vars = dict()
   for variable in ('KV_FULL', 'KV_MAJOR', 'KV_MINOR', 'KV_PATCH'):
-    result = run_command(config, "extract kernel version string from %s" % (context['build_env_file']), ['env', '-i', 'bash', '-c', "source %s && echo \"${%s}\"" % (context['build_env_file'], variable)])
+    result = lib.run_command(config, "extract kernel version string from %s" % (context['build_env_file']), ['env', '-i', 'bash', '-c', "source %s && echo \"${%s}\"" % (context['build_env_file'], variable)])
     if result == None:
       return "Could not extract kernel variable %s" % (variable)
     vars[variable] = result.strip()
@@ -121,9 +187,12 @@ def extract_source_variables(config, context):
   context['source_archive_dist'] = "%s/%s" % (kernel_series_distdir, source_archive_filename)
   sys.stdout.write("Kernel version: %s.%s.%s full: %s\n" % (kernel_version_major, kernel_version_minor, kernel_version_patch, kernel_version_string))
 
+  for key in 'kernel_version_string', 'kernel_series', 'kernel_series_distdir', 'source_archive_filename':
+    sys.stdout.write("%s: %s\n" % (key, context[key]))
   return ""
 
 # determine if source build is needed
+########################
 def check_source_build_needed(config, context):
   sys.stdout.write("Checking build environment file: %s\n" % (context['build_env_file']))
   if not os.path.isfile(context['build_env_file']):
@@ -131,7 +200,7 @@ def check_source_build_needed(config, context):
       sys.stdout.write("No kernel build environment file found but forced to rebuild source image anyway\n")
       return True
     sys.stdout.write("No kernel build environment file found\n")
-    result = run_command(config, "create kernel build environment file from ebuild %s" % (context['found_ebuild_filename']), ['ebuild', context['found_ebuild_filename'], 'clean', 'setup'])
+    result = lib.run_command(config, "create kernel build environment file from ebuild %s" % (context['found_ebuild_filename']), ['ebuild', context['found_ebuild_filename'], 'clean', 'setup'])
     if result == None:
       return True
     if not os.path.isfile(context['build_env_file']):
@@ -159,6 +228,7 @@ def check_source_build_needed(config, context):
   return False
 
 # determine if making build archive is needed
+########################
 def check_build_archive_needed(config, context):
   if os.path.isfile(context['source_archive_dist']):
     if os.path.isdir(context['build_kernel_dir']):
@@ -179,13 +249,14 @@ def check_build_archive_needed(config, context):
   return True
 
 # create archive for built source
+########################
 def make_build_source_archive(config, context):
   sys.stdout.write("Using build kernel directory: %s\n" % (context['build_kernel_dir']))
   env = dict(os.environ)
   env['SRCDIR'] = context['build_kernel_dir']
   env['DESTDIR'] = context['kernel_series_distdir']
   env['DESTFILENAME'] = context['source_archive_filename']
-  result = run_command(config, "create archive for kernel source %s" % (context['source_archive_dist']), ['bash', '-c', context['build_source_archive']], env)
+  result = lib.run_command(config, "create archive for kernel source %s" % (context['source_archive_dist']), ['bash', '-c', context['build_source_archive']], env)
   if result == None:
     return 3
   if not os.path.isfile(context['source_archive_dist']):
@@ -194,8 +265,9 @@ def make_build_source_archive(config, context):
   return 0
 
 # build kernel source
+########################
 def build_source(config, context):
-  result = run_command(config, "create kernel source from ebuild %s" % (context['found_ebuild_filename']), ['ebuild', context['found_ebuild_filename'], 'clean', 'install'])
+  result = lib.run_command(config, "create kernel source from ebuild %s" % (context['found_ebuild_filename']), ['ebuild', context['found_ebuild_filename'], 'clean', 'install'])
   if result == None:
     return 3
   if not os.path.isfile(context['build_env_file']):
@@ -214,11 +286,13 @@ def build_source(config, context):
   return 0
 
 # clear kernel source built
+########################
 def clear_source(config, context):
-  run_command(config, "clear kernel source for ebuild %s" % (context['found_ebuild_filename']), ['ebuild', context['found_ebuild_filename'], 'clean'])
+  lib.run_command(config, "clear kernel source for ebuild %s" % (context['found_ebuild_filename']), ['ebuild', context['found_ebuild_filename'], 'clean'])
   return 0
 
-# make source
+# make source (mksrc command)
+########################
 def make_source(config):
   context = dict()
   rc = get_source_package_config(config, context)
@@ -243,7 +317,12 @@ def make_source(config):
       return rc
   return None
 
-# prepare source
+# prepare source (prepare command)
+########################
 def prepare_source(config):
-  contetx = dict()
+  context = dict()
+  rc = get_source_package_config(config, context)
+  if rc:
+    return rc
+
   return None