Implemented mksrc command

.gitignore

@@ -1,3 +1,4 @@
 *~
 __pycache__
 archive
+distribution

README.md

@@ -17,6 +17,7 @@ ktool mksrc
 - load kernel source
 - add .config
 - make prepare
+- check if .config changed -> store it
 - store prepared kernel source
 
 ```bash

config/00defaults.yaml

@@ -6,3 +6,30 @@ distdir: distribution
 
 # kernel series to use (latest or x.y)
 kernels: latest
+
+# kernel package config selection
+package: mygentoo
+
+# build parameters
+ebuild:
+# cleanup after build
+  cleanup: no
+# portage build location
+##  prefix: /var/tmp/portage
+# format to use to store source, must exist under 'formats'
+  format: tarxz
+
+# format descriptions
+#   extension: extension to use for this format
+#   archive: command to create archive
+#     ${SRCDIR}: source directory to archive
+#     ${DESTDIR}: destination directory for archive file
+#     ${DESTFILENAME}: destination filename for archive file
+#   extract: command to extract archive
+#     ${SRCFILE}: source file for archive
+#     ${DESTDIR}: destination directory to extract to
+formats:
+  tarxz:
+    extension: tar.xz
+    archive: 'set -o pipefail && mkdir -p "${DESTDIR}" && tar cfC - "${SRCDIR}" . | xz -c - > "${DESTDIR}/.${DESTFILENAME}" && mv -f "${DESTDIR}/.${DESTFILENAME}" "${DESTDIR}/${DESTFILENAME}"'
+    extract: 'set -o pipefail && mkdir -p "${DESTDIR}" && xz -cd "${SRCFILE}" | tar xfC - "${DESTDIR}"'

config/01package.yaml

@@ -0,0 +1,4 @@
+# kernel package configs
+packages:
+  mygentoo:
+    pkg: sys-kernel/mygentoo-sources

configuration.py

@@ -3,6 +3,7 @@
 import os
 import sys
 import getopt
+import json
 from yaml import load, YAMLError
 try:
   from yaml import CSafeLoader as SafeLoader
@@ -10,6 +11,51 @@ except ImportError:
   from yaml import SafeLoader
 from pathlib import Path
 
+def merge_config(src, dest):
+  for key, value in src.items():
+    if isinstance(value, dict):
+      node = dest.setdefault(key, {})
+      merge_config(value, node)
+    else:
+      dest[key] = value
+  return dest
+
+def parse_config_set(params, a):
+  a = a.strip()
+  if a.startswith('['):
+    return "config parameters are not of list type"
+  if a.startswith('{'):
+    try:
+      value = json.loads(a)
+      merge_config(value, params)
+      return None
+    except json.JSONDecodeError as e:
+      return "failed to decode JSON struct: %s" % (e)
+
+  parts = a.partition('=')
+  key = parts[0].strip()
+  operator = parts[1].strip()
+  value = parts[2].strip()
+  if (operator != '='):
+    return "format required: key=jsonvalue or JSON struct"
+  if (key == ''):
+    return "format required: key=jsonvalue; key may not be an empty string"
+  try:
+    value = json.loads(value)
+  except json.JSONDecodeError as e:
+    return "failed to decode JSON value: %s" % (e)
+
+  key_list = key.split('.')
+  p = params
+  while True:
+    key = key_list.pop(0)
+    more = len(key_list) > 0
+    if not more:
+      p[key] = value
+      break
+    p = params.setdefault(key, {})
+  return None
+
 class Config():
   def __init__(self, scriptname, progver, commands):
     scriptname = os.path.realpath(scriptname)
@@ -21,6 +67,7 @@ class Config():
     self.help = False
     self.version = False
     self.verbose = False
+    self.force = False
     self.config = "config"
     self.params = {}
     # command
@@ -40,7 +87,9 @@ Options:
     -h, --help                     Print this help
     -V, --version                  Print version number
     -c, --config dir               Specify config directory to read [""" + self.config + """]
-    -s, --set key=value            Override config parameter key=value
+    -s, --set key=jsonvalue        Override config parameter 'key' with JSON format value
+    -s, --set jsonstruct           Override config parameters providing JSON struct
+    -f, --force                    Force recreating result
     -v, --verbose                  Print verbose messages
 """
     sys.stderr.write(usage)
@@ -49,8 +98,8 @@ Options:
 
   def parse(self, argv):
     try:
-      (opt, args) = getopt.gnu_getopt(argv[1:], "hVvc:s:", [
-        "help", "version", "verbose", "config=", "set="
+      (opt, args) = getopt.gnu_getopt(argv[1:], "hVvc:s:f", [
+        "help", "version", "verbose", "config=", "set=", "force"
         ])
     except getopt.GetoptError as e:
       sys.stderr.write("Error: %s\n" % str(e))
@@ -63,23 +112,21 @@ Options:
         self.version = True
       elif o == '-v' or o == "--verbose":
         self.verbose = True
+      elif o == '-f' or o == "--force":
+        self.force = True
       elif o == '-c' or o == "--config":
         if (a == ''):
-          sys.stderr.write("Error: option " + o + " requires a non-empty argument\n")
+          sys.stderr.write("Error: option %s requires a non-empty argument\n" % (o))
           self.usage(2)
         self.config = a
       elif o == '-s' or o == "--set":
-        parts = a.partition('=')
-        if (parts[1] != '='):
-          sys.stderr.write("Error: option " + o + " requires argument in the format key=value\n")
+        error = parse_config_set(self.params, a)
+        if error:
+          sys.stderr.write("Error: option %s argument is invalid: %s\n" % (o, error))
           self.usage(2)
-        if (parts[0] == ''):
-          sys.stderr.write("Error: option " + o + " requires argument in the format key=value; key may not be an empty string\n")
-          self.usage(2)
-        self.params[parts[0]] = parts[2]
 
     if self.version:
-      sys.stdout.write("Version: " + self.progver + "\n")
+      sys.stdout.write("Version: %s\n" % (self.progver))
     if self.help:
       self.usage(None)
     if self.version or self.help:
@@ -89,52 +136,90 @@ Options:
 
     for c in args:
       if self.cmd != None:
-        sys.stderr.write("Multiple commands specified: " + self.cmd + " vs " + c + "\n")
+        sys.stderr.write("Multiple commands specified: %s vs %s\n" % (self.cmd, c))
         self.usage(2)
       for cmd, _ in self.commands:
         if c == cmd:
           self.cmd = c
           break
       if self.cmd == None:
-        sys.stderr.write("Unknown command specified: " + c + "\n")
+        sys.stderr.write("Unknown command specified: %s\n" % (c))
         self.usage(2)
     if self.cmd == None:
       sys.stderr.write("Missing command\n")
       self.usage(2)
 
-    for k, v in self.params.items():
-      self.config_params[k] = v
+    merge_config(self.params, self.config_params)
     if self.verbose:
       sys.stdout.write("Config parameters:\n")
       for k, v in self.config_params.items():
-        sys.stdout.write("  " + k + "=" + str(v) + "\n")
-
-  def merge_config(self, src, dest):
-    for key, value in src.items():
-        if isinstance(value, dict):
-            node = dest.setdefault(key, {})
-            self.merge_config(value, node)
-        else:
-            dest[key] = value
-    return dest
+        sys.stdout.write("  %s=%s\n" % (k, str(v)))
 
   def parse_file(self, filepath):
     try:
       obj = load(open(filepath, 'r'), SafeLoader)
-      self.merge_config(obj, self.config_params)
+      merge_config(obj, self.config_params)
     except YAMLError as exc:
       sys.stdout.write("Error: reading configuration file failed: %s" % (exc))
 
   def load_config(self):
     configdir = self.config if self.config.startswith('/') else self.basedir + '/' + self.config
     if self.verbose:
-      sys.stdout.write("Reading config directory " + configdir + "\n")
+      sys.stdout.write("Reading config directory %s\n" % (configdir))
     if not Path(configdir).is_dir():
-      sys.stderr.write("Warning: config directory " + configdir + " does not exist\n")
+      sys.stderr.write("Warning: config directory %s does not exist\n" % (configdir))
       return
     with os.scandir(configdir) as it:
       for entry in it:
         if not entry.name.startswith('.') and (entry.name.endswith('.yml') or entry.name.endswith('.yaml')) and entry.is_file():
           if self.verbose:
-            sys.stdout.write("Parsing config file " + entry.path + "\n")
+            sys.stdout.write("Parsing config file %s\n" % (entry.path))
           self.parse_file(entry.path)
+
+  def get_config_param(self, path, mandatory = False, default = None, type = 'str'):
+    missing = False
+    lpath = path.split('.')
+    index = 0
+    d = self.config_params
+    while index < len(lpath):
+      key = lpath[index]
+      index += 1
+      if d == None:
+        missing = True
+        break
+      if not isinstance(d, dict):
+        missing = True
+        break
+      if key not in d:
+        missing = True
+        break
+      d = d[key]
+    if missing:
+      if mandatory:
+        sys.stdout.write("Configuration setting '%s' missing\n" % (path))
+      return default
+    if type == 'str':
+      return str(d)
+    if type == 'boolean':
+      if isinstance(d, (int, float)):
+        return int(d) != 0
+      value = str(d)
+      if value == '' or value == '0' or value == 'no' or value == 'No' or value == 'false' or value == 'False':
+        return False
+      return True
+    if type == 'int':
+      try:
+        return int(d)
+      except ValueError as e:
+        sys.stdout.write("Configuration '%s' is not an integer\n" % (path))
+        return default
+    if type == 'float':
+      try:
+        return float(d)
+      except ValueError as e:
+        sys.stdout.write("Configuration '%s' is not a float\n" % (path))
+        return default
+      except OverflowError as e:
+        sys.stdout.write("Configuration '%s' is overflows a float\n" % (path))
+        return default
+    return d

ktool

@@ -4,6 +4,7 @@
 import os
 import sys
 
+import tool
 import progver
 from configuration import Config
 
@@ -17,3 +18,10 @@ config = Config(__file__, progver.VERSION, [
   ['install', 'Install kernel']
 ])
 config.parse(sys.argv)
+rc = None
+if config.cmd == 'mksrc':
+  rc = tool.make_source(config)
+elif config.cmd == 'prepare':
+  rc = tool.prepare_source(config)
+if rc:
+  sys.exit(rc)

tool.py

@@ -0,0 +1,249 @@
+# -*- coding: UTF-8 -*-
+
+import os
+import re
+import sys
+import subprocess
+
+DISTDIR_DEFAULT = 'distribution'
+BUILD_PREFIX_DEFAULT = '/var/tmp/portage'
+PARSE_EBUILD_FILENAME = re.compile("^.*/([^/]+)/([^/]+)/([^/]+)\.ebuild$")
+
+# run command
+def run_command(config, task_name, argv, env = None):
+  sys.stdout.write("Running command '%s' to %s\n" % (' '.join(argv), task_name))
+  try:
+    result = subprocess.run(argv, stdout=subprocess.PIPE, env=env)
+    if config.verbose:
+      sys.stdout.write("Command output: %s\n" % (result.stdout.decode('utf-8')))
+    if result.returncode != 0:
+      sys.stdout.write("Failed to %s: exit code %d\n" % (task_name, result.returncode))
+      return None
+    return result.stdout.decode('utf-8')
+  except Exception as e:
+    sys.stdout.write("Failed to %s: %s\n" % (task_name, e))
+    return None
+
+# collect source making configuration
+def get_source_package_config(config, context):
+  config_pkg_name = config.get_config_param('package', mandatory=True)
+  if config_pkg_name == None:
+    return 3
+  config_pkg_id = 'packages.%s.pkg' % (config_pkg_name)
+  package_name = config.get_config_param(config_pkg_id, mandatory=True)
+  if package_name == None:
+    return 3
+
+  build_clean = config.get_config_param('ebuild.cleanup', default=True, type='boolean')
+  context['build_clean'] = build_clean
+
+  version_wanted = config.get_config_param('kernels', default='latest')
+  if version_wanted == 'latest':
+    version_wanted = ''
+  package_to_check = package_name if version_wanted == '' else '%s-%s*' % (package_name, version_wanted)
+  build_prefix = config.get_config_param('ebuild.prefix', default=BUILD_PREFIX_DEFAULT)
+  distdir = config.get_config_param('distdir', default=DISTDIR_DEFAULT)
+  context['package_to_check'] = package_to_check
+  context['build_prefix'] = build_prefix
+  context['distdir'] = distdir
+
+  build_source_format = config.get_config_param('ebuild.format', mandatory=True)
+  if build_source_format == None:
+    return 3
+  build_source_extension = config.get_config_param('formats.%s.extension' % (build_source_format), mandatory=True)
+  if build_source_extension == None:
+    return 3
+  build_source_archive = config.get_config_param('formats.%s.archive' % (build_source_format), mandatory=True)
+  if build_source_archive == None:
+    return 3
+  build_source_extract = config.get_config_param('formats.%s.extract' % (build_source_format), mandatory=True)
+  if build_source_extract == None:
+    return 3
+  context['build_source_format'] = build_source_format
+  context['build_source_extension'] = build_source_extension
+  context['build_source_archive'] = build_source_archive
+  context['build_source_extract'] = build_source_extract
+  return 0
+
+# determine source ebuild and version info
+def get_source_ebuild_info(config, context):
+  result = run_command(config, "get ebuild file for package %s" % (context['package_to_check']), ['equery', 'which', context['package_to_check']])
+  if result == None:
+    return 3
+  found_ebuild_filename = result.strip()
+
+  m = PARSE_EBUILD_FILENAME.match(found_ebuild_filename)
+  if not m:
+    sys.stdout.write("Could not parse ebuild file name: %s\n" % (found_ebuild_filename))
+    return 3
+  found_package_group = m.group(1)
+  found_package_name = m.group(2)
+  found_package_plus_version = m.group(3)
+  if not found_package_plus_version.startswith(found_package_name) or found_package_plus_version[len(found_package_name):len(found_package_name)+1] != '-':
+    sys.stdout.write("Inconsistent package name in ebuild file name: %s\n" % (found_ebuild_filename))
+    return 3
+  found_version = found_package_plus_version[len(found_package_name) + 1:]
+  found_package_fullname = '%s/%s-%s' % (found_package_group, found_package_name, found_version)
+  sys.stdout.write("Package version found: %s\n" % (found_package_fullname))
+
+  context['found_ebuild_filename'] = found_ebuild_filename
+  context['found_package_fullname'] = found_package_fullname
+  context['build_image_dir'] = "%s/%s/image" % (context['build_prefix'], found_package_fullname)
+  context['build_env_file'] = "%s/%s/temp/environment" % (context['build_prefix'], found_package_fullname)
+  return 0
+
+# extract build variables
+def extract_source_variables(config, context):
+  vars = dict()
+  for variable in ('KV_FULL', 'KV_MAJOR', 'KV_MINOR', 'KV_PATCH'):
+    result = run_command(config, "extract kernel version string from %s" % (context['build_env_file']), ['env', '-i', 'bash', '-c', "source %s && echo \"${%s}\"" % (context['build_env_file'], variable)])
+    if result == None:
+      return "Could not extract kernel variable %s" % (variable)
+    vars[variable] = result.strip()
+    if vars[variable] == '':
+      return "Could not extract kernel variable %s" % (variable)
+
+  kernel_version_string = vars['KV_FULL']
+  context['kernel_version_string'] = kernel_version_string
+  kernel_version_major = vars['KV_MAJOR']
+  context['kernel_version_major'] = kernel_version_major
+  kernel_version_minor = vars['KV_MINOR']
+  context['kernel_version_minor'] = kernel_version_minor
+  kernel_version_patch = vars['KV_PATCH']
+  context['kernel_version_patch'] = kernel_version_patch
+  kernel_series = "%s.%s" % (kernel_version_major, kernel_version_minor)
+  context['kernel_series'] = kernel_series
+  kernel_series_distdir = "%s/%s" % (context['distdir'], kernel_series)
+  context['kernel_series_distdir'] = kernel_series_distdir
+  context['build_kernel_dir'] = "%s/usr/src/linux-%s" % (context['build_image_dir'], kernel_version_string)
+  source_archive_filename = "%s.%s" % (kernel_version_string, context['build_source_extension'])
+  context['source_archive_filename'] = source_archive_filename
+  context['source_archive_dist'] = "%s/%s" % (kernel_series_distdir, source_archive_filename)
+  sys.stdout.write("Kernel version: %s.%s.%s full: %s\n" % (kernel_version_major, kernel_version_minor, kernel_version_patch, kernel_version_string))
+
+  return ""
+
+# determine if source build is needed
+def check_source_build_needed(config, context):
+  sys.stdout.write("Checking build environment file: %s\n" % (context['build_env_file']))
+  if not os.path.isfile(context['build_env_file']):
+    if config.force:
+      sys.stdout.write("No kernel build environment file found but forced to rebuild source image anyway\n")
+      return True
+    sys.stdout.write("No kernel build environment file found\n")
+    result = run_command(config, "create kernel build environment file from ebuild %s" % (context['found_ebuild_filename']), ['ebuild', context['found_ebuild_filename'], 'clean', 'setup'])
+    if result == None:
+      return True
+    if not os.path.isfile(context['build_env_file']):
+      sys.stdout.write("No kernel build environment file found after setup\n")
+      return True
+  else:
+    if config.force:
+      sys.stdout.write("Existing kernel build environment file found but forced to rebuild source image\n")
+      return True
+  error = extract_source_variables(config, context)
+  if error:
+    sys.stdout.write("%s\n" % (error))
+    return True
+  if os.path.isfile(context['source_archive_dist']):
+    return False
+  sys.stdout.write("Checking build image location: %s\n" % (context['build_image_dir']))
+  if not os.path.isdir(context['build_image_dir']):
+    sys.stdout.write("No kernel source image found\n")
+    return True
+  sys.stdout.write("Checking build kernel directory: %s\n" % (context['build_kernel_dir']))
+  if not os.path.isdir(context['build_kernel_dir']):
+    sys.stdout.write("No kernel directory found\n")
+    return True
+  sys.stdout.write("Existing kernel directory found\n")
+  return False
+
+# determine if making build archive is needed
+def check_build_archive_needed(config, context):
+  if os.path.isfile(context['source_archive_dist']):
+    if os.path.isdir(context['build_kernel_dir']):
+      try:
+        dest_mtime = os.path.getmtime(context['source_archive_dist'])
+      except OSError as e:
+        return True
+      try:
+        src_mtime = os.path.getmtime(context['build_kernel_dir'])
+      except OSError as e:
+        return True
+      sys.stdout.write("%s vs %s\n" % (src_mtime, dest_mtime))
+      if src_mtime > dest_mtime:
+        sys.stdout.write("Existing source archive older than built image\n")
+        return True
+    sys.stdout.write("Existing source archive found: %s\n" % (context['source_archive_dist']))
+    return False
+  return True
+
+# create archive for built source
+def make_build_source_archive(config, context):
+  sys.stdout.write("Using build kernel directory: %s\n" % (context['build_kernel_dir']))
+  env = dict(os.environ)
+  env['SRCDIR'] = context['build_kernel_dir']
+  env['DESTDIR'] = context['kernel_series_distdir']
+  env['DESTFILENAME'] = context['source_archive_filename']
+  result = run_command(config, "create archive for kernel source %s" % (context['source_archive_dist']), ['bash', '-c', context['build_source_archive']], env)
+  if result == None:
+    return 3
+  if not os.path.isfile(context['source_archive_dist']):
+    sys.stdout.write("Kernel source archive file not found after creation\n")
+    return 3
+  return 0
+
+# build kernel source
+def build_source(config, context):
+  result = run_command(config, "create kernel source from ebuild %s" % (context['found_ebuild_filename']), ['ebuild', context['found_ebuild_filename'], 'clean', 'install'])
+  if result == None:
+    return 3
+  if not os.path.isfile(context['build_env_file']):
+    sys.stdout.write("No kernel build environment file found after ebuild command\n")
+    return 3
+  error = extract_source_variables(config, context)
+  if error:
+    sys.stdout.write("%s after ebuild command\n" % (error))
+    return 3
+  if not os.path.isdir(context['build_image_dir']):
+    sys.stdout.write("No kernel source image found after ebuild command\n")
+    return 3
+  if not os.path.isdir(context['build_kernel_dir']):
+    sys.stdout.write("No kernel directory found after ebuild command\n")
+    return 3
+  return 0
+
+# clear kernel source built
+def clear_source(config, context):
+  run_command(config, "clear kernel source for ebuild %s" % (context['found_ebuild_filename']), ['ebuild', context['found_ebuild_filename'], 'clean'])
+  return 0
+
+# make source
+def make_source(config):
+  context = dict()
+  rc = get_source_package_config(config, context)
+  if rc:
+    return rc
+  rc = get_source_ebuild_info(config, context)
+  if rc:
+    return rc
+  build = check_source_build_needed(config, context)
+  if build:
+    rc = build_source(config, context)
+    if rc:
+      return rc
+  make = check_build_archive_needed(config, context)
+  if make:
+    rc = make_build_source_archive(config, context)
+    if rc:
+      return rc
+  if context['build_clean']:
+    rc = clear_source(config, context)
+    if rc:
+      return rc
+  return None
+
+# prepare source
+def prepare_source(config):
+  contetx = dict()
+  return None