SymbX/systemd
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Apply "ruff format" and "ruff check --fix" for all python scripts (#40758)

Browse Source
This commit is contained in:
Yu Watanabe
2026-05-18 07:26:11 +09:00
committed by GitHub
parent 4b2a88fe07 2cdf552762
commit 9b768b5bf5
63 changed files with 6025 additions and 4106 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.dir-locals.el
+8
View File
@@ -22,6 +22,14 @@
(meson-mode . ((meson-indent-basic . 8)))
(sh-mode . ((sh-indentation . 4)))
(awk-mode . ((c-basic-offset . 8)))
(python-mode . ((indent-tabs-mode . nil)
(tab-width . 4)
(fill-column . 109)
(python-indent-def-block-scale . 1)))
(python-ts-mode . ((indent-tabs-mode . nil)
(tab-width . 4)
(fill-column . 109)
(python-indent-def-block-scale . 1)))
(nil . ((indent-tabs-mode . nil)
(tab-width . 8)
(fill-column . 79))) )
.github/workflows/linter.yml
+8 -3
View File
@@ -63,16 +63,21 @@ jobs:
- name: Run ruff format
run: |
mkosi box -- ruff --version
if ! mkosi box -- ruff format --check src/core/generate-bpf-delegate-configs.py src/boot/generate-hwids-section.py src/test/generate-sym-test.py src/ukify/ukify.py test/integration-tests/integration-test-wrapper.py
if ! mkosi box -- ruff format --check
then
echo "Please run 'ruff format' on the above files or apply the diffs below manually"
mkosi box -- ruff format --check --quiet --diff src/core/generate-bpf-delegate-configs.py src/boot/generate-hwids-section.py src/test/generate-sym-test.py src/ukify/ukify.py test/integration-tests/integration-test-wrapper.py
mkosi box -- ruff format --check --quiet --diff
fi
- name: Run ruff check
run: |
mkosi box -- ruff --version
mkosi box -- ruff check src/core/generate-bpf-delegate-configs.py src/boot/generate-hwids-section.py src/test/generate-sym-test.py src/ukify/ukify.py test/integration-tests/integration-test-wrapper.py
if ! mkosi box -- ruff check
then
echo "Please fix the errors shown by 'ruff check' on the above files or add noqa comments where appropriate."
echo "Diffs for fixes that can be programmatically generated are shown below:"
mkosi box -- ruff check --diff
fi
- name: Configure meson
run: mkosi box -- env CC=clang CXX=clang++ meson setup build
.ycm_extra_conf.py
+21 -24
View File
@@ -43,11 +43,11 @@ used by the 'systemd' project.
import glob
import os
import ycm_core
SOURCE_EXTENSIONS = (".C", ".cpp", ".cxx", ".cc", ".c", ".m", ".mm")
HEADER_EXTENSIONS = (".H", ".h", ".hxx", ".hpp", ".hh")
SOURCE_EXTENSIONS = ('.C', '.cpp', '.cxx', '.cc', '.c', '.m', '.mm')
HEADER_EXTENSIONS = ('.H', '.h', '.hxx', '.hpp', '.hh')
def DirectoryOfThisScript():
@@ -69,19 +69,18 @@ def GuessBuildDirectory():
containing '.ninja_log' file two levels above the current directory;
returns this single directory only if there is one candidate.
"""
result = os.path.join(DirectoryOfThisScript(), "build")
result = os.path.join(DirectoryOfThisScript(), 'build')
if os.path.exists(result):
return result
result = glob.glob(os.path.join(DirectoryOfThisScript(),
"..", "..", "*", ".ninja_log"))
result = glob.glob(os.path.join(DirectoryOfThisScript(), '..', '..', '*', '.ninja_log'))
if not result:
return ""
return ''
if 1 != len(result):
return ""
return ''
return os.path.split(result[0])[0]
@@ -106,9 +105,7 @@ def TraverseByDepth(root, include_extensions):
# print(subdirs)
if include_extensions:
get_ext = os.path.splitext
subdir_extensions = {
get_ext(f)[-1] for f in file_list if get_ext(f)[-1]
}
subdir_extensions = {get_ext(f)[-1] for f in file_list if get_ext(f)[-1]}
if subdir_extensions & include_extensions:
result.add(root_dir)
else:
@@ -119,11 +116,11 @@ def TraverseByDepth(root, include_extensions):
return result
_project_src_dir = os.path.join(DirectoryOfThisScript(), "src")
_include_dirs_set = TraverseByDepth(_project_src_dir, frozenset({".h"}))
_project_src_dir = os.path.join(DirectoryOfThisScript(), 'src')
_include_dirs_set = TraverseByDepth(_project_src_dir, frozenset({'.h'}))
flags = [
"-x",
"c"
'-x',
'c',
# The following flags are partially redundant due to the existence of
# 'compile_commands.json'.
# '-Wall',
@@ -135,7 +132,7 @@ flags = [
]
for include_dir in _include_dirs_set:
flags.append("-I" + include_dir)
flags.append('-I' + include_dir)
# Set this to the absolute path to the folder (NOT the file!) containing the
# compile_commands.json file to use that instead of 'flags'. See here for
@@ -165,13 +162,13 @@ def MakeRelativePathsInFlagsAbsolute(flags, working_directory):
return list(flags)
new_flags = []
make_next_absolute = False
path_flags = ["-isystem", "-I", "-iquote", "--sysroot="]
path_flags = ['-isystem', '-I', '-iquote', '--sysroot=']
for flag in flags:
new_flag = flag
if make_next_absolute:
make_next_absolute = False
if not flag.startswith("/"):
if not flag.startswith('/'):
new_flag = os.path.join(working_directory, flag)
for path_flag in path_flags:
@@ -180,7 +177,7 @@ def MakeRelativePathsInFlagsAbsolute(flags, working_directory):
break
if flag.startswith(path_flag):
path = flag[len(path_flag):]
path = flag[len(path_flag) :]
new_flag = path_flag + os.path.join(working_directory, path)
break
@@ -213,8 +210,7 @@ def GetCompilationInfoForFile(filename):
for extension in SOURCE_EXTENSIONS:
replacement_file = basename + extension
if os.path.exists(replacement_file):
compilation_info = \
database.GetCompilationInfoForFile(replacement_file)
compilation_info = database.GetCompilationInfoForFile(replacement_file)
if compilation_info.compiler_flags_:
return compilation_info
return None
@@ -238,13 +234,14 @@ def FlagsForFile(filename, **kwargs):
final_flags = MakeRelativePathsInFlagsAbsolute(
compilation_info.compiler_flags_,
compilation_info.compiler_working_dir_)
compilation_info.compiler_working_dir_,
)
else:
relative_to = DirectoryOfThisScript()
final_flags = MakeRelativePathsInFlagsAbsolute(flags, relative_to)
return {
"flags": final_flags,
"do_cache": True
'flags': final_flags,
'do_cache': True,
}
hwdb.d/acpi-update.py
+2
View File
@@ -5,6 +5,7 @@ from csv import reader
# pylint: disable=consider-using-with
def read_table(filename):
table = list(reader(open(filename, newline='')))
table = table[1:] # Skip header
@@ -15,6 +16,7 @@ def read_table(filename):
# a mistake, strip it.
print(f'\nacpi:{row[1].strip()}*:\n ID_VENDOR_FROM_DATABASE={row[0].strip()}')
print('''\
# This file is part of systemd.
#
hwdb.d/ids_parser.py
+77 -38
View File
@@ -3,11 +3,24 @@
import re
import sys
from pyparsing import (Word, White, Literal, Regex,
LineEnd, SkipTo,
ZeroOrMore, OneOrMore, Combine, Optional, Suppress,
Group, ParserElement,
stringEnd, pythonStyleComment)
from pyparsing import (
Combine,
Group,
LineEnd,
Literal,
OneOrMore,
Optional,
ParserElement,
Regex,
SkipTo,
Suppress,
White,
Word,
ZeroOrMore,
pythonStyleComment,
stringEnd,
)
EOL = LineEnd().suppress()
NUM1 = Word('0123456789abcdefABCDEF', exact=1)
@@ -22,7 +35,9 @@ text_eol = lambda name: Regex(r'[^\n]+')(name) + EOL
ParserElement.setDefaultWhitespaceChars(' \n')
def klass_grammar():
# fmt: off
klass_line = Literal('C ').suppress() + NUM2('klass') + text_eol('text')
subclass_line = TAB + NUM2('subclass') + text_eol('text')
protocol_line = TAB + TAB + NUM2('protocol') + text_eol('name')
@@ -32,9 +47,12 @@ def klass_grammar():
klass = (klass_line('KLASS') -
ZeroOrMore(Group(subclass)('SUBCLASSES*')
^ COMMENTLINE.suppress()))
# fmt: on
return klass
def usb_ids_grammar():
# fmt: off
vendor_line = NUM4('vendor') + text_eol('text')
device_line = TAB + NUM4('device') + text_eol('text')
interface_line = TAB + TAB + NUM4('interface') + NUM4('interface2') + text_eol('text')
@@ -55,11 +73,14 @@ def usb_ids_grammar():
grammar = OneOrMore(Group(vendor)('VENDORS*')
^ Group(klass)('CLASSES*')
^ other_group.suppress() ^ commentgroup) + stringEnd()
# fmt: on
grammar.parseWithTabs()
return grammar
def pci_ids_grammar():
# fmt: off
vendor_line = NUM4('vendor') + text_eol('text')
device_line = TAB + NUM4('device') + text_eol('text')
subvendor_line = TAB + TAB + NUM4('a') + White(' ') + NUM4('b') + text_eol('name')
@@ -75,11 +96,14 @@ def pci_ids_grammar():
grammar = OneOrMore(Group(vendor)('VENDORS*')
^ Group(klass)('CLASSES*')
^ commentgroup) + stringEnd()
# fmt: on
grammar.parseWithTabs()
return grammar
def sdio_ids_grammar():
# fmt: off
vendor_line = NUM4('vendor') + text_eol('text')
device_line = TAB + NUM4('device') + text_eol('text')
vendor = (vendor_line('VENDOR') +
@@ -91,11 +115,14 @@ def sdio_ids_grammar():
grammar = OneOrMore(Group(vendor)('VENDORS*')
^ Group(klass)('CLASSES*')
^ commentgroup) + stringEnd()
# fmt: on
grammar.parseWithTabs()
return grammar
def oui_grammar(type):
# fmt: off
prefix_line = (Combine(NUM2 - Suppress('-') - NUM2 - Suppress('-') - NUM2)('prefix')
- Literal('(hex)') - text_eol('text'))
if type == 'small':
@@ -115,18 +142,23 @@ def oui_grammar(type):
grammar = (Literal('OUI') + text_eol('header')
+ text_eol('header') + text_eol('header') + EMPTYLINE
+ OneOrMore(Group(vendor)('VENDORS*')) + stringEnd())
# fmt: on
grammar.parseWithTabs()
return grammar
def header(file, *sources):
print('''\
sep = ' ' if len(sources) == 1 else '\n# '
joined = sep + sep.join(sources)
print(
f'''\
# This file is part of systemd.
#
# Data imported from:{}{}'''.format(' ' if len(sources) == 1 else '\n# ',
'\n# '.join(sources)),
file=file)
# Data imported from:{joined}''',
file=file,
)
def add_item(items, key, value):
if key in items:
@@ -134,6 +166,7 @@ def add_item(items, key, value):
else:
items[key] = value
def usb_vendor_model(p):
items = {}
@@ -147,19 +180,19 @@ def usb_vendor_model(p):
text = vendor_dev.text.strip()
add_item(items, (vendor, device), text)
with open('20-usb-vendor-model.hwdb', 'wt') as out:
with open('20-usb-vendor-model.hwdb', 'w') as out:
header(out, 'http://www.linux-usb.org/usb.ids')
for key in sorted(items):
if len(key) == 1:
p, n = 'usb:v{}*', 'VENDOR'
else:
p, n = 'usb:v{}p{}*', 'MODEL',
print('', p.format(*key),
f' ID_{n}_FROM_DATABASE={items[key]}', sep='\n', file=out)
p, n = 'usb:v{}p{}*', 'MODEL'
print('', p.format(*key), f' ID_{n}_FROM_DATABASE={items[key]}', sep='\n', file=out)
print(f'Wrote {out.name}')
def usb_classes(p):
items = {}
@@ -182,7 +215,7 @@ def usb_classes(p):
if klass != '00' and not re.match(r'(\?|None|Unused)\s*$', text):
add_item(items, (klass, subclass, protocol), text)
with open('20-usb-classes.hwdb', 'wt') as out:
with open('20-usb-classes.hwdb', 'w') as out:
header(out, 'http://www.linux-usb.org/usb.ids')
for key in sorted(items):
@@ -192,11 +225,11 @@ def usb_classes(p):
p, n = 'usb:v*p*d*dc{}dsc{}*', 'SUBCLASS'
else:
p, n = 'usb:v*p*d*dc{}dsc{}dp{}*', 'PROTOCOL'
print('', p.format(*key),
f' ID_USB_{n}_FROM_DATABASE={items[key]}', sep='\n', file=out)
print('', p.format(*key), f' ID_USB_{n}_FROM_DATABASE={items[key]}', sep='\n', file=out)
print(f'Wrote {out.name}')
def pci_vendor_model(p):
items = {}
@@ -215,12 +248,12 @@ def pci_vendor_model(p):
sub_model = subvendor_group.b.upper()
sub_text = subvendor_group.name.strip()
if sub_text.startswith(text):
sub_text = sub_text[len(text):].lstrip()
sub_text = sub_text[len(text) :].lstrip()
if sub_text:
sub_text = f' ({sub_text})'
add_item(items, (vendor, device, sub_vendor, sub_model), text + sub_text)
with open('20-pci-vendor-model.hwdb', 'wt') as out:
with open('20-pci-vendor-model.hwdb', 'w') as out:
header(out, 'http://pci-ids.ucw.cz/v2.2/pci.ids')
for key in sorted(items):
@@ -230,11 +263,11 @@ def pci_vendor_model(p):
p, n = 'pci:v0000{}d0000{}*', 'MODEL'
else:
p, n = 'pci:v0000{}d0000{}sv0000{}sd0000{}*', 'MODEL'
print('', p.format(*key),
f' ID_{n}_FROM_DATABASE={items[key]}', sep='\n', file=out)
print('', p.format(*key), f' ID_{n}_FROM_DATABASE={items[key]}', sep='\n', file=out)
print(f'Wrote {out.name}')
def pci_classes(p):
items = {}
@@ -253,7 +286,7 @@ def pci_classes(p):
text = protocol_group.name.strip()
add_item(items, (klass, subclass, protocol), text)
with open('20-pci-classes.hwdb', 'wt') as out:
with open('20-pci-classes.hwdb', 'w') as out:
header(out, 'http://pci-ids.ucw.cz/v2.2/pci.ids')
for key in sorted(items):
@@ -263,11 +296,11 @@ def pci_classes(p):
p, n = 'pci:v*d*sv*sd*bc{}sc{}*', 'SUBCLASS'
else:
p, n = 'pci:v*d*sv*sd*bc{}sc{}i{}*', 'INTERFACE'
print('', p.format(*key),
f' ID_PCI_{n}_FROM_DATABASE={items[key]}', sep='\n', file=out)
print('', p.format(*key), f' ID_PCI_{n}_FROM_DATABASE={items[key]}', sep='\n', file=out)
print(f'Wrote {out.name}')
def sdio_vendor_model(p):
items = {}
@@ -281,7 +314,7 @@ def sdio_vendor_model(p):
text = device_group.text.strip()
add_item(items, (vendor, device), text)
with open('20-sdio-vendor-model.hwdb', 'wt') as out:
with open('20-sdio-vendor-model.hwdb', 'w') as out:
header(out, 'hwdb.d/sdio.ids')
for key in sorted(items):
@@ -289,11 +322,11 @@ def sdio_vendor_model(p):
p, n = 'sdio:c*v{}*', 'VENDOR'
else:
p, n = 'sdio:c*v{}d{}*', 'MODEL'
print('', p.format(*key),
f' ID_{n}_FROM_DATABASE={items[key]}', sep='\n', file=out)
print('', p.format(*key), f' ID_{n}_FROM_DATABASE={items[key]}', sep='\n', file=out)
print(f'Wrote {out.name}')
def sdio_classes(p):
items = {}
@@ -302,16 +335,21 @@ def sdio_classes(p):
text = klass_group.text.strip()
add_item(items, klass, text)
with open('20-sdio-classes.hwdb', 'wt') as out:
with open('20-sdio-classes.hwdb', 'w') as out:
header(out, 'hwdb.d/sdio.ids')
for klass in sorted(items):
print(f'',
f'sdio:c{klass}v*d*',
f' ID_SDIO_CLASS_FROM_DATABASE={items[klass]}', sep='\n', file=out)
print(
'',
f'sdio:c{klass}v*d*',
f' ID_SDIO_CLASS_FROM_DATABASE={items[klass]}',
sep='\n',
file=out,
)
print(f'Wrote {out.name}')
# MAC Address Block Large/Medium/Small
# Large MA-L 24/24 bit (OUI)
# Medium MA-M 28/20 bit (OUI prefix owned by IEEE)
@@ -338,19 +376,20 @@ def oui(p1, p2, p3):
key = prefix + start if end else prefix
add_item(items, key, text)
with open('20-OUI.hwdb', 'wt') as out:
header(out,
'https://services13.ieee.org/RST/standards-ra-web/rest/assignments/download/?registry=MA-L&format=txt',
'https://services13.ieee.org/RST/standards-ra-web/rest/assignments/download/?registry=MA-M&format=txt',
'https://services13.ieee.org/RST/standards-ra-web/rest/assignments/download/?registry=MA-S&format=txt')
with open('20-OUI.hwdb', 'w') as out:
header(
out,
'https://services13.ieee.org/RST/standards-ra-web/rest/assignments/download/?registry=MA-L&format=txt',
'https://services13.ieee.org/RST/standards-ra-web/rest/assignments/download/?registry=MA-M&format=txt',
'https://services13.ieee.org/RST/standards-ra-web/rest/assignments/download/?registry=MA-S&format=txt',
)
for pattern in sorted(items):
print(f'',
f'OUI:{pattern}*',
f' ID_OUI_FROM_DATABASE={items[pattern]}', sep='\n', file=out)
print('', f'OUI:{pattern}*', f' ID_OUI_FROM_DATABASE={items[pattern]}', sep='\n', file=out)
print(f'Wrote {out.name}')
if __name__ == '__main__':
args = sys.argv[1:]
hwdb.d/parse_hwdb.py
+210 -169
View File
@@ -26,16 +26,31 @@
# SOFTWARE.
import glob
import os
import string
import sys
import os
try:
from pyparsing import (Word, White, Literal, ParserElement, Regex, LineEnd,
OneOrMore, Combine, Or, Optional, Suppress, Group,
nums, alphanums, printables,
stringEnd, pythonStyleComment,
ParseBaseException)
from pyparsing import (
Combine,
Group,
LineEnd,
Literal,
OneOrMore,
Optional,
Or,
ParseBaseException,
ParserElement,
Regex,
Suppress,
White,
Word,
alphanums,
nums,
printables,
pythonStyleComment,
stringEnd,
)
except ImportError:
print('pyparsing is not available')
sys.exit(77)
@@ -57,11 +72,7 @@ except ImportError:
ecodes = None
print('WARNING: evdev is not available')
try:
from functools import lru_cache
except ImportError:
# don't do caching on old python
lru_cache = lambda: (lambda f: f)
from functools import lru_cache
EOL = LineEnd().suppress()
EMPTYLINE = LineEnd()
@@ -72,78 +83,90 @@ SIGNED_REAL = Combine(Optional(Word('-+')) + REAL)
UDEV_TAG = Word(string.ascii_uppercase, alphanums + '_')
# Those patterns are used in type-specific matches
TYPES = {'mouse': ('usb', 'bluetooth', 'ps2', '*'),
'evdev': ('name', 'atkbd', 'input'),
'fb': ('pci', 'vmbus'),
'id-input': ('modalias', 'bluetooth', 'i2c', 'usb'),
'touchpad': ('i8042', 'rmi', 'bluetooth', 'usb'),
'joystick': ('i8042', 'rmi', 'bluetooth', 'usb'),
'keyboard': ('name', ),
'sensor': ('modalias',
'accel-base',
'accel-display',
'accel-camera',
'proximity-palmrest',
'proximity-palmrest-left',
'proximity-palmrest-right',
'proximity-lap',
'proximity-wifi',
'proximity-lte',
'proximity-wifi-lte',
'proximity-wifi-left',
'proximity-wifi-right',
),
'ieee1394-unit-function' : ('node', ),
'camera': ('usb'),
}
TYPES = {
'mouse': ('usb', 'bluetooth', 'ps2', '*'),
'evdev': ('name', 'atkbd', 'input'),
'fb': ('pci', 'vmbus'),
'id-input': ('modalias', 'bluetooth', 'i2c', 'usb'),
'touchpad': ('i8042', 'rmi', 'bluetooth', 'usb'),
'joystick': ('i8042', 'rmi', 'bluetooth', 'usb'),
'keyboard': ('name',),
'sensor': (
'modalias',
'accel-base',
'accel-display',
'accel-camera',
'proximity-palmrest',
'proximity-palmrest-left',
'proximity-palmrest-right',
'proximity-lap',
'proximity-wifi',
'proximity-lte',
'proximity-wifi-lte',
'proximity-wifi-left',
'proximity-wifi-right',
),
'ieee1394-unit-function': ('node',),
'camera': ('usb'),
}
# Patterns that are used to set general properties on a device
GENERAL_MATCHES = {'acpi',
'bluetooth',
'dmi',
'ieee1394',
'OUI',
'pci',
'sdio',
'tpm2',
'usb',
'vmbus',
}
GENERAL_MATCHES = {
'acpi',
'bluetooth',
'dmi',
'ieee1394',
'OUI',
'pci',
'sdio',
'tpm2',
'usb',
'vmbus',
}
def upperhex_word(length):
return Word(nums + 'ABCDEF', exact=length)
@lru_cache()
@lru_cache
def hwdb_grammar():
ParserElement.setDefaultWhitespaceChars('')
prefix = Or(category + ':' + Or(conn) + ':'
for category, conn in TYPES.items())
prefix = Or(category + ':' + Or(conn) + ':' for category, conn in TYPES.items())
matchline_typed = Combine(prefix + Word(printables + ' ' + '®'))
matchline_general = Combine(Or(GENERAL_MATCHES) + ':' + Word(printables + ' ' + '®'))
matchline = (matchline_typed | matchline_general) + EOL
propertyline = (White(' ', exact=1).suppress() +
Combine(UDEV_TAG - '=' - Optional(Word(alphanums + '_=:@*.!-;, "/?&'))
- Optional(pythonStyleComment)) +
EOL)
propertyline = (
White(' ', exact=1).suppress()
+ Combine(
UDEV_TAG - '=' - Optional(Word(alphanums + '_=:@*.!-;, "/?&')) - Optional(pythonStyleComment)
)
+ EOL
)
propertycomment = White(' ', exact=1) + pythonStyleComment + EOL
group = (OneOrMore(matchline('MATCHES*') ^ COMMENTLINE.suppress()) -
OneOrMore(propertyline('PROPERTIES*') ^ propertycomment.suppress()) -
(EMPTYLINE ^ stringEnd()).suppress())
group = (
OneOrMore(matchline('MATCHES*') ^ COMMENTLINE.suppress())
- OneOrMore(propertyline('PROPERTIES*') ^ propertycomment.suppress())
- (EMPTYLINE ^ stringEnd()).suppress()
)
commentgroup = OneOrMore(COMMENTLINE).suppress() - EMPTYLINE.suppress()
grammar = OneOrMore(Group(group)('GROUPS*') ^ commentgroup) + stringEnd()
return grammar
@lru_cache()
@lru_cache
def property_grammar():
ParserElement.setDefaultWhitespaceChars(' ')
dpi_setting = Group(Optional('*')('DEFAULT') + INTEGER('DPI') + Optional(Suppress('@') + INTEGER('HZ')))('SETTINGS*')
dpi_setting = Group(Optional('*')('DEFAULT') + INTEGER('DPI') + Optional(Suppress('@') + INTEGER('HZ')))(
'SETTINGS*'
)
mount_matrix_row = SIGNED_REAL + ',' + SIGNED_REAL + ',' + SIGNED_REAL
mount_matrix = Group(mount_matrix_row + ';' + mount_matrix_row + ';' + mount_matrix_row)('MOUNT_MATRIX')
xkb_setting = Optional(Word(alphanums + '+-/@._'))
@@ -153,132 +176,139 @@ def property_grammar():
# Although this set doesn't cover all of characters in database entries, it's enough for test targets.
name_literal = Word(printables + ' ')
props = (('MOUSE_DPI', Group(OneOrMore(dpi_setting))),
('MOUSE_WHEEL_CLICK_ANGLE', INTEGER),
('MOUSE_WHEEL_CLICK_ANGLE_HORIZONTAL', INTEGER),
('MOUSE_WHEEL_CLICK_COUNT', INTEGER),
('MOUSE_WHEEL_CLICK_COUNT_HORIZONTAL', INTEGER),
('ID_INPUT_3D_MOUSE', zero_one),
('ID_AUTOSUSPEND', zero_one),
('ID_AUTOSUSPEND_DELAY_MS', INTEGER),
('ID_AV_PRODUCTION_CONTROLLER', zero_one),
('ID_AV_LIGHTS', zero_one),
('ID_PERSIST', zero_one),
('ID_PDA', zero_one),
('ID_INPUT', id_input_setting),
('ID_INPUT_ACCELEROMETER', id_input_setting),
('ID_INPUT_JOYSTICK', id_input_setting),
('ID_INPUT_KEY', id_input_setting),
('ID_INPUT_KEYBOARD', id_input_setting),
('ID_INPUT_MOUSE', id_input_setting),
('ID_INPUT_POINTINGSTICK', id_input_setting),
('ID_INPUT_SWITCH', id_input_setting),
('ID_INPUT_TABLET', id_input_setting),
('ID_INPUT_TABLET_PAD', id_input_setting),
('ID_INPUT_TOUCHPAD', id_input_setting),
('ID_INPUT_TOUCHSCREEN', id_input_setting),
('ID_INPUT_TRACKBALL', id_input_setting),
('ID_SIGNAL_ANALYZER', zero_one),
('ID_MAKER_TOOL', zero_one),
('ID_HARDWARE_WALLET', zero_one),
('ID_SOFTWARE_RADIO', zero_one),
('ID_MM_DEVICE_IGNORE', zero_one),
('ID_NET_AUTO_LINK_LOCAL_ONLY', zero_one),
('POINTINGSTICK_SENSITIVITY', INTEGER),
('ID_INTEGRATION', Or(('internal', 'external'))),
('ID_INPUT_TOUCHPAD_INTEGRATION', Or(('internal', 'external'))),
('XKB_FIXED_LAYOUT', xkb_setting),
('XKB_FIXED_VARIANT', xkb_setting),
('XKB_FIXED_MODEL', xkb_setting),
('KEYBOARD_LED_NUMLOCK', Literal('0')),
('KEYBOARD_LED_CAPSLOCK', Literal('0')),
('ACCEL_MOUNT_MATRIX', mount_matrix),
('ACCEL_LOCATION', Or(('display', 'base'))),
('PROXIMITY_NEAR_LEVEL', INTEGER),
('IEEE1394_UNIT_FUNCTION_MIDI', zero_one),
('IEEE1394_UNIT_FUNCTION_AUDIO', zero_one),
('IEEE1394_UNIT_FUNCTION_VIDEO', zero_one),
('ID_VENDOR_FROM_DATABASE', name_literal),
('ID_MODEL_FROM_DATABASE', name_literal),
('ID_TAG_MASTER_OF_SEAT', Literal('1')),
('ID_INFRARED_CAMERA', zero_one),
('ID_CAMERA_DIRECTION', Or(('front', 'rear'))),
('SOUND_FORM_FACTOR', Or(('internal', 'webcam', 'speaker', 'headphone', 'headset', 'handset', 'microphone'))),
('ID_SYS_VENDOR_IS_RUBBISH', zero_one),
('ID_PRODUCT_NAME_IS_RUBBISH', zero_one),
('ID_PRODUCT_VERSION_IS_RUBBISH', zero_one),
('ID_BOARD_VERSION_IS_RUBBISH', zero_one),
('ID_PRODUCT_SKU_IS_RUBBISH', zero_one),
('ID_CHASSIS_ASSET_TAG_IS_RUBBISH', zero_one),
('ID_CHASSIS', name_literal),
('ID_SYSFS_ATTRIBUTE_MODEL', name_literal),
('ID_NET_NAME_FROM_DATABASE', name_literal),
('ID_NET_NAME_INCLUDE_DOMAIN', zero_one),
('TPM2_BROKEN_NVPCR', zero_one),
('IMDS_VENDOR', name_literal),
('IMDS_TOKEN_URL', name_literal),
('IMDS_REFRESH_HEADER_NAME', name_literal),
('IMDS_DATA_URL', name_literal),
('IMDS_DATA_URL_SUFFIX', name_literal),
('IMDS_TOKEN_HEADER_NAME', name_literal),
('IMDS_EXTRA_HEADER', name_literal),
('IMDS_ADDRESS_IPV4', name_literal),
('IMDS_ADDRESS_IPV6', name_literal),
('IMDS_KEY_HOSTNAME', name_literal),
('IMDS_KEY_REGION', name_literal),
('IMDS_KEY_ZONE', name_literal),
('IMDS_KEY_IPV4_PUBLIC', name_literal),
('IMDS_KEY_IPV6_PUBLIC', name_literal),
('IMDS_KEY_SSH_KEY', name_literal),
('IMDS_KEY_USERDATA', name_literal),
('IMDS_KEY_USERDATA_BASE', name_literal),
('IMDS_KEY_USERDATA_BASE64', name_literal),
)
fixed_props = [Literal(name)('NAME') - Suppress('=') - val('VALUE')
for name, val in props]
kbd_props = [Regex(r'KEYBOARD_KEY_[0-9a-f]+')('NAME')
- Suppress('=') -
Group('!' ^ (Optional('!') - Word(alphanums + '_')))('VALUE')
]
abs_props = [Regex(r'EVDEV_ABS_[0-9a-f]{2}')('NAME')
- Suppress('=') -
Word('-' + nums + ':')('VALUE')
]
props = (
('MOUSE_DPI', Group(OneOrMore(dpi_setting))),
('MOUSE_WHEEL_CLICK_ANGLE', INTEGER),
('MOUSE_WHEEL_CLICK_ANGLE_HORIZONTAL', INTEGER),
('MOUSE_WHEEL_CLICK_COUNT', INTEGER),
('MOUSE_WHEEL_CLICK_COUNT_HORIZONTAL', INTEGER),
('ID_INPUT_3D_MOUSE', zero_one),
('ID_AUTOSUSPEND', zero_one),
('ID_AUTOSUSPEND_DELAY_MS', INTEGER),
('ID_AV_PRODUCTION_CONTROLLER', zero_one),
('ID_AV_LIGHTS', zero_one),
('ID_PERSIST', zero_one),
('ID_PDA', zero_one),
('ID_INPUT', id_input_setting),
('ID_INPUT_ACCELEROMETER', id_input_setting),
('ID_INPUT_JOYSTICK', id_input_setting),
('ID_INPUT_KEY', id_input_setting),
('ID_INPUT_KEYBOARD', id_input_setting),
('ID_INPUT_MOUSE', id_input_setting),
('ID_INPUT_POINTINGSTICK', id_input_setting),
('ID_INPUT_SWITCH', id_input_setting),
('ID_INPUT_TABLET', id_input_setting),
('ID_INPUT_TABLET_PAD', id_input_setting),
('ID_INPUT_TOUCHPAD', id_input_setting),
('ID_INPUT_TOUCHSCREEN', id_input_setting),
('ID_INPUT_TRACKBALL', id_input_setting),
('ID_SIGNAL_ANALYZER', zero_one),
('ID_MAKER_TOOL', zero_one),
('ID_HARDWARE_WALLET', zero_one),
('ID_SOFTWARE_RADIO', zero_one),
('ID_MM_DEVICE_IGNORE', zero_one),
('ID_NET_AUTO_LINK_LOCAL_ONLY', zero_one),
('POINTINGSTICK_SENSITIVITY', INTEGER),
('ID_INTEGRATION', Or(('internal', 'external'))),
('ID_INPUT_TOUCHPAD_INTEGRATION', Or(('internal', 'external'))),
('XKB_FIXED_LAYOUT', xkb_setting),
('XKB_FIXED_VARIANT', xkb_setting),
('XKB_FIXED_MODEL', xkb_setting),
('KEYBOARD_LED_NUMLOCK', Literal('0')),
('KEYBOARD_LED_CAPSLOCK', Literal('0')),
('ACCEL_MOUNT_MATRIX', mount_matrix),
('ACCEL_LOCATION', Or(('display', 'base'))),
('PROXIMITY_NEAR_LEVEL', INTEGER),
('IEEE1394_UNIT_FUNCTION_MIDI', zero_one),
('IEEE1394_UNIT_FUNCTION_AUDIO', zero_one),
('IEEE1394_UNIT_FUNCTION_VIDEO', zero_one),
('ID_VENDOR_FROM_DATABASE', name_literal),
('ID_MODEL_FROM_DATABASE', name_literal),
('ID_TAG_MASTER_OF_SEAT', Literal('1')),
('ID_INFRARED_CAMERA', zero_one),
('ID_CAMERA_DIRECTION', Or(('front', 'rear'))),
(
'SOUND_FORM_FACTOR',
Or(('internal', 'webcam', 'speaker', 'headphone', 'headset', 'handset', 'microphone')),
),
('ID_SYS_VENDOR_IS_RUBBISH', zero_one),
('ID_PRODUCT_NAME_IS_RUBBISH', zero_one),
('ID_PRODUCT_VERSION_IS_RUBBISH', zero_one),
('ID_BOARD_VERSION_IS_RUBBISH', zero_one),
('ID_PRODUCT_SKU_IS_RUBBISH', zero_one),
('ID_CHASSIS_ASSET_TAG_IS_RUBBISH', zero_one),
('ID_CHASSIS', name_literal),
('ID_SYSFS_ATTRIBUTE_MODEL', name_literal),
('ID_NET_NAME_FROM_DATABASE', name_literal),
('ID_NET_NAME_INCLUDE_DOMAIN', zero_one),
('TPM2_BROKEN_NVPCR', zero_one),
('IMDS_VENDOR', name_literal),
('IMDS_TOKEN_URL', name_literal),
('IMDS_REFRESH_HEADER_NAME', name_literal),
('IMDS_DATA_URL', name_literal),
('IMDS_DATA_URL_SUFFIX', name_literal),
('IMDS_TOKEN_HEADER_NAME', name_literal),
('IMDS_EXTRA_HEADER', name_literal),
('IMDS_ADDRESS_IPV4', name_literal),
('IMDS_ADDRESS_IPV6', name_literal),
('IMDS_KEY_HOSTNAME', name_literal),
('IMDS_KEY_REGION', name_literal),
('IMDS_KEY_ZONE', name_literal),
('IMDS_KEY_IPV4_PUBLIC', name_literal),
('IMDS_KEY_IPV6_PUBLIC', name_literal),
('IMDS_KEY_SSH_KEY', name_literal),
('IMDS_KEY_USERDATA', name_literal),
('IMDS_KEY_USERDATA_BASE', name_literal),
('IMDS_KEY_USERDATA_BASE64', name_literal),
)
fixed_props = [Literal(name)('NAME') - Suppress('=') - val('VALUE') for name, val in props]
kbd_props = [
Regex(r'KEYBOARD_KEY_[0-9a-f]+')('NAME')
- Suppress('=')
- Group('!' ^ (Optional('!') - Word(alphanums + '_')))('VALUE')
]
abs_props = [Regex(r'EVDEV_ABS_[0-9a-f]{2}')('NAME') - Suppress('=') - Word('-' + nums + ':')('VALUE')]
grammar = Or(fixed_props + kbd_props + abs_props) + EOL
return grammar
ERROR = False
def error(fmt, *args, **kwargs):
global ERROR
ERROR = True
print(fmt.format(*args, **kwargs))
def convert_properties(group):
matches = [m[0] for m in group.MATCHES]
props = [p[0] for p in group.PROPERTIES]
return matches, props
def parse(fname):
grammar = hwdb_grammar()
try:
with open(fname, 'r', encoding='UTF-8') as f:
with open(fname, encoding='UTF-8') as f:
parsed = grammar.parseFile(f)
except ParseBaseException as e:
error('Cannot parse {}: {}', fname, e)
return []
return [convert_properties(g) for g in parsed.GROUPS]
def check_matches(groups):
matches = sum((group[0] for group in groups), [])
# This is a partial check. The other cases could be also done, but those
# three are the most commonly wrong.
grammars = {
'bluetooth' : 'v' + upperhex_word(4) + Optional('p' + upperhex_word(4) + Optional(':')) + '*',
'usb' : 'v' + upperhex_word(4) + Optional('p' + upperhex_word(4) + Optional(':')) + '*',
'pci' : 'v' + upperhex_word(8) + Optional('d' + upperhex_word(8) + Optional(':')) + '*',
'bluetooth': 'v' + upperhex_word(4) + Optional('p' + upperhex_word(4) + Optional(':')) + '*',
'usb': 'v' + upperhex_word(4) + Optional('p' + upperhex_word(4) + Optional(':')) + '*',
'pci': 'v' + upperhex_word(8) + Optional('d' + upperhex_word(8) + Optional(':')) + '*',
}
for match in matches:
@@ -301,11 +331,13 @@ def check_matches(groups):
error('Match {!r} is duplicated', match)
prev = match
def check_one_default(prop, settings):
defaults = [s for s in settings if s.DEFAULT]
if len(defaults) > 1:
error('More than one star entry: {!r}', prop)
def check_one_mount_matrix(prop, value):
numbers = [s for s in value if s not in {';', ','}]
if len(numbers) != 9:
@@ -316,28 +348,33 @@ def check_one_mount_matrix(prop, value):
error('Wrong accel matrix: {!r}', prop)
bad_x, bad_y, bad_z = max(numbers[0:3]) == 0, max(numbers[3:6]) == 0, max(numbers[6:9]) == 0
if bad_x or bad_y or bad_z:
error('Mount matrix is all zero in {} row: {!r}',
'x' if bad_x else ('y' if bad_y else 'z'),
prop)
error('Mount matrix is all zero in {} row: {!r}', 'x' if bad_x else ('y' if bad_y else 'z'), prop)
def check_one_keycode(value):
if value != '!' and ecodes is not None:
key = 'KEY_' + value.upper()
if not (key in ecodes or
value.upper() in ecodes or
# new keys added in kernel 5.5
'KBD_LCD_MENU' in key):
if not (
key in ecodes
or value.upper() in ecodes
# new keys added in kernel 5.5
or 'KBD_LCD_MENU' in key
): # fmt: skip
error('Keycode {} unknown', key)
def check_wheel_clicks(properties):
pairs = (('MOUSE_WHEEL_CLICK_COUNT_HORIZONTAL', 'MOUSE_WHEEL_CLICK_COUNT'),
('MOUSE_WHEEL_CLICK_ANGLE_HORIZONTAL', 'MOUSE_WHEEL_CLICK_ANGLE'),
('MOUSE_WHEEL_CLICK_COUNT_HORIZONTAL', 'MOUSE_WHEEL_CLICK_ANGLE_HORIZONTAL'),
('MOUSE_WHEEL_CLICK_COUNT', 'MOUSE_WHEEL_CLICK_ANGLE'))
pairs = (
('MOUSE_WHEEL_CLICK_COUNT_HORIZONTAL', 'MOUSE_WHEEL_CLICK_COUNT'),
('MOUSE_WHEEL_CLICK_ANGLE_HORIZONTAL', 'MOUSE_WHEEL_CLICK_ANGLE'),
('MOUSE_WHEEL_CLICK_COUNT_HORIZONTAL', 'MOUSE_WHEEL_CLICK_ANGLE_HORIZONTAL'),
('MOUSE_WHEEL_CLICK_COUNT', 'MOUSE_WHEEL_CLICK_ANGLE'),
)
for pair in pairs:
if pair[0] in properties and pair[1] not in properties:
error('{} requires {} to be specified', *pair)
def check_properties(groups):
grammar = property_grammar()
for _, props in groups:
@@ -364,6 +401,7 @@ def check_properties(groups):
check_wheel_clicks(seen_props)
def print_summary(fname, groups):
n_matches = sum(len(matches) for matches, props in groups)
n_props = sum(len(props) for matches, props in groups)
@@ -372,12 +410,15 @@ def print_summary(fname, groups):
if n_matches == 0 or n_props == 0:
print(f'{fname}: no matches or props')
if __name__ == '__main__':
args = sys.argv[1:] or sorted([
os.path.dirname(sys.argv[0]) + '/20-dmi-id.hwdb',
os.path.dirname(sys.argv[0]) + '/20-net-ifname.hwdb',
*glob.glob(os.path.dirname(sys.argv[0]) + '/[678][0-9]-*.hwdb'),
])
args = sys.argv[1:] or sorted(
[
os.path.dirname(sys.argv[0]) + '/20-dmi-id.hwdb',
os.path.dirname(sys.argv[0]) + '/20-net-ifname.hwdb',
*glob.glob(os.path.dirname(sys.argv[0]) + '/[678][0-9]-*.hwdb'),
]
)
for fname in args:
groups = parse(fname)
man/90-rearrange-path.py
+8 -6
View File
@@ -17,6 +17,7 @@ interesting generators.
import os
import pathlib
def rearrange_bin_sbin(path):
"""Make sure any pair of …/bin, …/sbin directories is in this order
@@ -27,15 +28,16 @@ def rearrange_bin_sbin(path):
for i in range(len(items)):
if 'sbin' in items[i].parts:
ind = items[i].parts.index('sbin')
bin = pathlib.Path(*items[i].parts[:ind], 'bin', *items[i].parts[ind+1:])
if bin in items[i+1:]:
j = i + 1 + items[i+1:].index(bin)
bin = pathlib.Path(*items[i].parts[:ind], 'bin', *items[i].parts[ind + 1 :])
if bin in items[i + 1 :]:
j = i + 1 + items[i + 1 :].index(bin)
items[i], items[j] = items[j], items[i]
return ':'.join(p.as_posix() for p in items)
if __name__ == '__main__':
path = os.environ['PATH'] # This should be always set.
# If it is not, we will just crash, which is OK too.
# This should be always set. If it is not, we will just crash, which is OK too.
path = os.environ['PATH']
new = rearrange_bin_sbin(path)
if new != path:
print('PATH={}'.format(new))
print(f'PATH={new}')
man/check-os-release-simple.py
+2 -2
View File
@@ -2,11 +2,11 @@
# SPDX-License-Identifier: MIT-0
import platform
os_release = platform.freedesktop_os_release()
pretty_name = os_release.get('PRETTY_NAME', 'Linux')
print(f'Running on {pretty_name!r}')
if 'fedora' in [os_release.get('ID', 'linux'),
*os_release.get('ID_LIKE', '').split()]:
if 'fedora' in [os_release.get('ID', 'linux'), *os_release.get('ID_LIKE', '').split()]:
print('Looks like Fedora!')
man/check-os-release.py
+4 -4
View File
@@ -5,6 +5,7 @@ import ast
import re
import sys
def read_os_release():
try:
filename = '/etc/os-release'
@@ -23,14 +24,13 @@ def read_os_release():
val = ast.literal_eval(val)
yield name, val
else:
print(f'{filename}:{line_number}: bad line {line!r}',
file=sys.stderr)
print(f'{filename}:{line_number}: bad line {line!r}', file=sys.stderr)
os_release = dict(read_os_release())
pretty_name = os_release.get('PRETTY_NAME', 'Linux')
print(f'Running on {pretty_name!r}')
if 'debian' in [os_release.get('ID', 'linux'),
*os_release.get('ID_LIKE', '').split()]:
if 'debian' in [os_release.get('ID', 'linux'), *os_release.get('ID_LIKE', '').split()]:
print('Looks like Debian!')
man/notify-selfcontained-example.py
+28 -20
View File
@@ -18,49 +18,56 @@ import time
reloading = False
terminating = False
def notify(message):
if not message:
raise ValueError("notify() requires a message")
raise ValueError('notify() requires a message')
socket_path = os.environ.get("NOTIFY_SOCKET")
socket_path = os.environ.get('NOTIFY_SOCKET')
if not socket_path:
return
if socket_path[0] not in ("/", "@"):
raise OSError(errno.EAFNOSUPPORT, "Unsupported socket type")
if socket_path[0] not in ('/', '@'):
raise OSError(errno.EAFNOSUPPORT, 'Unsupported socket type')
# Handle abstract socket.
if socket_path[0] == "@":
socket_path = "\0" + socket_path[1:]
if socket_path[0] == '@':
socket_path = '\0' + socket_path[1:]
with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM | socket.SOCK_CLOEXEC) as sock:
sock.connect(socket_path)
sock.sendall(message)
def notify_ready():
notify(b"READY=1")
notify(b'READY=1')
def notify_reloading():
microsecs = time.clock_gettime_ns(time.CLOCK_MONOTONIC) // 1000
notify(f"RELOADING=1\nMONOTONIC_USEC={microsecs}".encode())
notify(f'RELOADING=1\nMONOTONIC_USEC={microsecs}'.encode())
def notify_stopping():
notify(b"STOPPING=1")
notify(b'STOPPING=1')
def reload(signum, frame):
global reloading
reloading = True
def terminate(signum, frame):
global terminating
terminating = True
def main():
print("Doing initial setup")
print('Doing initial setup')
global reloading, terminating
# Set up signal handlers.
print("Setting up signal handlers")
print('Setting up signal handlers')
signal.signal(signal.SIGHUP, reload)
signal.signal(signal.SIGINT, terminate)
signal.signal(signal.SIGTERM, terminate)
@@ -68,13 +75,13 @@ def main():
# Do any other setup work here.
# Once all setup is done, signal readiness.
print("Done setting up")
print('Done setting up')
notify_ready()
print("Starting loop")
print('Starting loop')
while not terminating:
if reloading:
print("Reloading")
print('Reloading')
reloading = False
# Support notifying the manager when reloading configuration.
@@ -86,19 +93,20 @@ def main():
# Do some reconfiguration work here.
print("Done reloading")
print('Done reloading')
notify_ready()
# Do the real work here ...
print("Sleeping for five seconds")
print('Sleeping for five seconds')
time.sleep(5)
print("Terminating")
print('Terminating')
notify_stopping()
if __name__ == "__main__":
if __name__ == '__main__':
sys.stdout.reconfigure(line_buffering=True)
print("Starting app")
print('Starting app')
main()
print("Stopped app")
print('Stopped app')
ruff.toml
+16 -2
View File
@@ -1,6 +1,20 @@
target-version = "py39"
line-length = 109
lint.select = ["E", "F", "I", "UP"]
# This file is imported from an external project.
extend-exclude = ["tools/chromiumos/gen_autosuspend_rules.py"]
[lint]
select = ["E", "F", "I", "Q", "UP"]
# E501: line-too-long
# E731: lambda-assignment
ignore = ["E501", "E731"]
[format]
quote-style = "single"
# The formatter prefers double quotes for multiline quotes,
# Hence, let's make the formatter not change quotations.
quote-style = "preserve"
[lint.flake8-quotes]
inline-quotes = "single"
multiline-quotes = "single"
src/basic/filesystem-sets.py
+109 -98
View File
@@ -124,12 +124,12 @@ NAME_TO_MAGIC = {
'z3fold': ['Z3FOLD_MAGIC'],
'zonefs': ['ZONEFS_MAGIC'],
'zsmalloc': ['ZSMALLOC_MAGIC'],
}
} # fmt: skip
# System magics are sometimes not unique, because file systems got new
# revisions or got renamed. Let's prefer newer over older here, and thus ignore
# the old names.
OBSOLETE_NAMES = {
OBSOLETE_NAMES = {
'cpuset', # magic taken over by cgroupfs
'devtmpfs', # not a file system of its own, but just a "named superblock" of tmpfs
'ext2', # ext4 is the newest revision of ext2 + ext3
@@ -142,115 +142,116 @@ OBSOLETE_NAMES = {
'nfs', # nfs4 is the newest revision of nfs
'pvfs2', # orangefs is the new name of pvfs2
'smb3', # smb3 is an alias for cifs
}
} # fmt: skip
FILESYSTEM_SETS = [
(
"@basic-api",
"Basic filesystem API",
"cgroup",
"cgroup2",
"devpts",
"devtmpfs",
"mqueue",
"proc",
"sysfs",
'@basic-api',
'Basic filesystem API',
'cgroup',
'cgroup2',
'devpts',
'devtmpfs',
'mqueue',
'proc',
'sysfs',
),
(
"@anonymous",
"Anonymous inodes",
"anon_inodefs",
"pipefs",
"sockfs",
'@anonymous',
'Anonymous inodes',
'anon_inodefs',
'pipefs',
'sockfs',
),
(
"@application",
"Application virtual filesystems",
"autofs",
"fuse",
"overlay",
'@application',
'Application virtual filesystems',
'autofs',
'fuse',
'overlay',
),
(
"@auxiliary-api",
"Auxiliary filesystem API",
"binfmt_misc",
"configfs",
"efivarfs",
"fusectl",
"hugetlbfs",
"rpc_pipefs",
"securityfs",
'@auxiliary-api',
'Auxiliary filesystem API',
'binfmt_misc',
'configfs',
'efivarfs',
'fusectl',
'hugetlbfs',
'rpc_pipefs',
'securityfs',
),
(
"@common-block",
"Common block device filesystems",
"btrfs",
"erofs",
"exfat",
"ext4",
"f2fs",
"iso9660",
"ntfs3",
"squashfs",
"udf",
"vfat",
"xfs",
'@common-block',
'Common block device filesystems',
'btrfs',
'erofs',
'exfat',
'ext4',
'f2fs',
'iso9660',
'ntfs3',
'squashfs',
'udf',
'vfat',
'xfs',
),
(
"@historical-block",
"Historical block device filesystems",
"ext2",
"ext3",
"minix",
'@historical-block',
'Historical block device filesystems',
'ext2',
'ext3',
'minix',
),
(
"@network",
"Well-known network filesystems",
"afs",
"ceph",
"cifs",
"gfs",
"gfs2",
"ncp",
"ncpfs",
"nfs",
"nfs4",
"ocfs2",
"orangefs",
"pvfs2",
"smb3",
"smbfs",
'@network',
'Well-known network filesystems',
'afs',
'ceph',
'cifs',
'gfs',
'gfs2',
'ncp',
'ncpfs',
'nfs',
'nfs4',
'ocfs2',
'orangefs',
'pvfs2',
'smb3',
'smbfs',
),
(
"@privileged-api",
"Privileged filesystem API",
"bpf",
"debugfs",
"pstore",
"tracefs",
'@privileged-api',
'Privileged filesystem API',
'bpf',
'debugfs',
'pstore',
'tracefs',
),
(
"@security",
"Security/MAC API VFS",
"apparmorfs",
"selinuxfs",
"smackfs",
'@security',
'Security/MAC API VFS',
'apparmorfs',
'selinuxfs',
'smackfs',
),
(
"@temporary",
"Temporary filesystems",
"ramfs",
"tmpfs",
'@temporary',
'Temporary filesystems',
'ramfs',
'tmpfs',
),
(
"@known",
"All known filesystems declared in the kernel",
'@known',
'All known filesystems declared in the kernel',
*NAME_TO_MAGIC.keys(),
),
]
def generate_gperf():
print("""\
print('''\
/* SPDX-License-Identifier: LGPL-2.1-or-later */
%{
#if __GNUC__ >= 15
@@ -275,12 +276,13 @@ struct FilesystemMagic;
%omit-struct-type
%struct-type
%includes
%%""")
%%''')
for name, magics in NAME_TO_MAGIC.items():
print(f"{name + ',':16} {{{', '.join(magics)}}}")
print(f'{name + ",":16} {{{", ".join(magics)}}}')
def generate_fs_type_to_string():
print("""\
print('''\
#include <linux/magic.h>
#include "filesystems.h"
@@ -290,7 +292,7 @@ def generate_fs_type_to_string():
#define PROJECT_FILE __FILE__
const char* fs_type_to_string(statfs_f_type_t magic) {
switch (magic) {""")
switch (magic) {''')
for name, magics in NAME_TO_MAGIC.items():
if name in OBSOLETE_NAMES:
@@ -299,10 +301,11 @@ const char* fs_type_to_string(statfs_f_type_t magic) {
print(f' case (statfs_f_type_t) {magic}:')
print(f' return "{name}";')
print("""\
print('''\
}
return NULL;
}""")
}''')
def generate_fs_in_group():
print('bool fs_in_group(const struct statfs *st, FilesystemGroups fs_group) {')
@@ -312,14 +315,14 @@ def generate_fs_in_group():
magics = sorted(set(sum((NAME_TO_MAGIC[fs] for fs in filesystems), start=[])))
enum = 'FILESYSTEM_SET_' + name[1:].upper().replace('-', '_')
print(f' case {enum}:')
opts = '\n || '.join(f'F_TYPE_EQUAL(st->f_type, {magic})'
for magic in magics)
opts = '\n || '.join(f'F_TYPE_EQUAL(st->f_type, {magic})' for magic in magics)
print(f' return {opts};')
print(' default: assert_not_reached();')
print(' }')
print('}')
def generate_filesystem_sets():
print('const FilesystemSet filesystem_sets[_FILESYSTEM_SET_MAX] = {')
@@ -329,37 +332,45 @@ def generate_filesystem_sets():
print(f' [{enum}] = {{')
print(f' .name = "{name}",')
print(f' .help = "{desc}",')
print(f' .value =')
print(' .value =')
for filesystem in filesystems:
print(f' "{filesystem}\\0"')
print(' },')
print('};')
def magic_defines():
cpp = os.environ['CPP'].split()
out = subprocess.check_output(
[*cpp, '-dM', '-include', 'linux/magic.h', '-'],
stdin=subprocess.DEVNULL,
text=True)
text=True,
)
for line in out.splitlines():
_, name, *rest = line.split()
if ('_MAGIC' in name
and rest and rest[0].startswith('0x')
and name not in {
if (
'_MAGIC' in name
and rest
and rest[0].startswith('0x')
and name
not in {
'STACK_END_MAGIC',
'MTD_INODE_FS_MAGIC',
'FUTEXFS_SUPER_MAGIC',
'CRAMFS_MAGIC_WEND',
}):
}
):
yield name
def check():
kernel_magics = set(magic_defines())
our_magics = set(sum(NAME_TO_MAGIC.values(), start=[]))
extra = kernel_magics - our_magics
if extra:
sys.exit(f"kernel knows additional filesystem magics: {', '.join(sorted(extra))}")
sys.exit(f'kernel knows additional filesystem magics: {", ".join(sorted(extra))}')
if __name__ == '__main__':
for arg in sys.argv[1:]:
src/boot/generate-hwids-section.py
+4 -4
View File
@@ -16,13 +16,13 @@ BYTES_PER_LINE = 16
hwids = ukify.parse_hwid_dir(Path(sys.argv[1]))
print(
"""/* SPDX-License-Identifier: LGPL-2.1-or-later */
'''/* SPDX-License-Identifier: LGPL-2.1-or-later */
#include <stddef.h>
#include <stdint.h>
// NOLINTNEXTLINE(misc-use-internal-linkage)
const uint8_t hwids_section_data[] = {
""",
''',
end='',
)
@@ -34,9 +34,9 @@ for i, b in enumerate(hwids):
print('')
print(
"""};
'''};
// NOLINTNEXTLINE(misc-use-internal-linkage)
const size_t hwids_section_len =""",
const size_t hwids_section_len =''',
f'{len(hwids)};',
)
src/bpf/merge-bpf-compdb.py
+8 -6
View File
@@ -11,7 +11,7 @@ def main() -> int:
sep = sys.argv.index('--')
sources = sys.argv[1:sep]
command = sys.argv[sep + 1:]
command = sys.argv[sep + 1 :]
db_path = os.path.join(build_root, 'compile_commands.json')
try:
@@ -24,11 +24,13 @@ def main() -> int:
db = [entry for entry in db if entry['file'] not in sources_set]
for source in sources:
db.append({
'directory': build_root,
'file': source,
'arguments': [source if a == '@INPUT@' else a for a in command],
})
db.append(
{
'directory': build_root,
'file': source,
'arguments': [source if a == '@INPUT@' else a for a in command],
}
)
with open(db_path, 'w') as f:
json.dump(db, f, indent=2)
src/core/generate-bpf-delegate-configs.py
+2 -2
View File
@@ -30,12 +30,12 @@ with open(header) as file:
enumName = ''
if output == 'doc':
print("""\
print('''\
<?xml version="1.0"?>
<!DOCTYPE bpf-delegates PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
<para>
""")
''')
for line in file:
line = line.strip()
src/fuzz/fuzz-bootspec-gen.py
+4 -6
View File
@@ -4,14 +4,12 @@
"""Generate sample input for fuzz-bootspec"""
import json
import os
import sys
from pathlib import Path
config = open(sys.argv[1]).read()
loader = [entry for entry in open(sys.argv[2], encoding='utf-16-le').read().split('\0')
if len(entry) > 2] # filter out fluff from bad decoding
entries = [(os.path.basename(name), open(name).read())
for name in sys.argv[3:]]
config = Path(sys.argv[1]).read_text()
loader = [entry for entry in Path(sys.argv[2]).read_text(encoding='utf-16-le').split('\0') if len(entry) > 2]
entries = [(Path(name).name, Path(name).read_text()) for name in sys.argv[3:]]
data = {
'config': config,
src/include/override/sys/generate-syscall.py
+39 -24
View File
@@ -1,8 +1,8 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: LGPL-2.1-or-later
import sys
import functools
import sys
# We only generate numbers for a dozen or so syscalls
SYSCALLS = [
@@ -12,24 +12,29 @@ SYSCALLS = [
'quotactl_fd', # defined in glibc header since glibc-2.35
'removexattrat',
'setxattrat',
]
] # fmt: skip
def dictify(f):
def wrap(*args, **kwargs):
return dict(f(*args, **kwargs))
return functools.update_wrapper(wrap, f)
@dictify
def parse_syscall_table(filename):
print(f'Reading {filename}…')
for line in open(filename):
items = line.split()
if len(items) >= 2:
yield items[0], int(items[1])
with open(filename) as f:
for line in f:
items = line.split()
if len(items) >= 2:
yield items[0], int(items[1])
def parse_syscall_tables(filenames):
return {filename.split('-')[-1][:-4]: parse_syscall_table(filename)
for filename in filenames}
return {filename.split('-')[-1][:-4]: parse_syscall_table(filename) for filename in filenames}
HEADER = '''\
/* SPDX-License-Identifier: LGPL-2.1-or-later
@@ -108,12 +113,15 @@ DEF_TEMPLATE_B = '''\
# else
# define systemd_NR_{syscall} {nr_x86_64}
# endif
# elif !defined(missing_arch_template)
%s
# endif
'''
DEF_TEMPLATE_C = '''\
# elif !defined(missing_arch_template)
# warning "{syscall}() syscall number is unknown for your architecture"
# endif
'''
DEF_TEMPLATE_D = '''\
/* may be an (invalid) negative number due to libseccomp, see PR 13319 */
# if defined __NR_{syscall} && __NR_{syscall} >= 0
@@ -130,23 +138,29 @@ static_assert(__NR_{syscall} == systemd_NR_{syscall}, "");
# endif
#endif'''
DEF_TEMPLATE = (DEF_TEMPLATE_A +
DEF_TEMPLATE_B % '# warning "{syscall}() syscall number is unknown for your architecture"' +
DEF_TEMPLATE_C)
DEF_TEMPLATE = DEF_TEMPLATE_A + DEF_TEMPLATE_B + DEF_TEMPLATE_C + DEF_TEMPLATE_D
ARCH_CHECK = '''\
ARCH_CHECK_A = '''\
/* Note: if this code looks strange, this is because it is derived from the same
* template as the per-syscall blocks below. */
''' + '\n'.join(line for line in DEF_TEMPLATE_B.splitlines()
if ' define ' not in line) % '''\
'''
ARCH_CHECK_B = '\n'.join(line for line in DEF_TEMPLATE_B.splitlines() if ' define ' not in line)
ARCH_CHECK_C = '''\
# else
# warning "Current architecture is missing from the template"
# define missing_arch_template 1'''
# define missing_arch_template 1
# endif'''
ARCH_CHECK = ARCH_CHECK_A + ARCH_CHECK_B + ARCH_CHECK_C
def print_syscall_def(syscall, tables, out):
mappings = {f'nr_{arch}':t.get(syscall, -1)
for arch, t in tables.items()}
print(DEF_TEMPLATE.format(syscall=syscall, **mappings),
file=out)
mappings = {f'nr_{arch}': t.get(syscall, -1) for arch, t in tables.items()}
print(DEF_TEMPLATE.format(syscall=syscall, **mappings), file=out)
def print_syscall_defs(syscalls, tables, out):
print(HEADER, file=out)
@@ -154,12 +168,13 @@ def print_syscall_defs(syscalls, tables, out):
for syscall in syscalls:
print_syscall_def(syscall, tables, out)
if __name__ == '__main__':
output_file = sys.argv[1]
arch_files = sys.argv[2:]
out = open(output_file, 'wt')
tables = parse_syscall_tables(arch_files)
print_syscall_defs(SYSCALLS, tables, out)
with open(output_file, 'w') as out:
print_syscall_defs(SYSCALLS, tables, out)
print(f'Wrote {output_file}')
src/include/override/sys/syscall.h
+1 -1
View File
@@ -49,7 +49,7 @@
# if defined(__ILP32__)
# else
# endif
# elif !defined(missing_arch_template)
# else
# warning "Current architecture is missing from the template"
# define missing_arch_template 1
# endif
src/journal-remote/log-generator.py
+14 -12
View File
@@ -1,8 +1,8 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: LGPL-2.1-or-later
import sys
import argparse
import sys
PARSER = argparse.ArgumentParser()
PARSER.add_argument('n', type=int)
@@ -12,7 +12,7 @@ PARSER.add_argument('-d', '--data-size', type=int, default=4000)
PARSER.add_argument('--data-type', choices={'random', 'simple'})
OPTIONS = PARSER.parse_args()
template = """\
template = '''\
__CURSOR=s=6863c726210b4560b7048889d8ada5c5;i=3e931;b=f446871715504074bf7049ef0718fa93;m={m:x};t=4fd05c
__REALTIME_TIMESTAMP={realtime_ts}
__MONOTONIC_TIMESTAMP={monotonic_ts}
@@ -30,7 +30,7 @@ SYSLOG_PID=25721
_PID=25721
_SOURCE_REALTIME_TIMESTAMP={source_realtime_ts}
DATA={data}
"""
'''
priority = 3
facility = 6
@@ -51,14 +51,16 @@ for i in range(OPTIONS.n):
data = '{:0{}}'.format(counter, OPTIONS.data_size)
counter += 1
entry = template.format(m=0x198603b12d7 + i,
realtime_ts=1404101101501873 + i,
monotonic_ts=1753961140951 + i,
source_realtime_ts=1404101101483516 + i,
priority=priority,
facility=facility,
message=message,
data=data)
entry = template.format(
m=0x198603B12D7 + i,
realtime_ts=1404101101501873 + i,
monotonic_ts=1753961140951 + i,
source_realtime_ts=1404101101483516 + i,
priority=priority,
facility=facility,
message=message,
data=data,
)
bytes += len(entry)
@@ -69,4 +71,4 @@ for i in range(OPTIONS.n):
if OPTIONS.dots:
print(file=sys.stderr)
print('Wrote {} bytes'.format(bytes), file=sys.stderr)
print(f'Wrote {bytes} bytes', file=sys.stderr)
src/shared/ethtool-link-mode.py
+1 -1
View File
@@ -7,7 +7,7 @@ import subprocess
import sys
OVERRIDES = {
'autoneg' : 'autonegotiation',
'autoneg': 'autonegotiation',
}
mode = sys.argv[1]
src/shared/generate-dns_type-gperf.py
+7 -9
View File
@@ -1,26 +1,24 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: LGPL-2.1-or-later
"""Generate %-from-name.gperf from %-list.txt
"""
"""Generate %-from-name.gperf from %-list.txt"""
import sys
name, prefix, input = sys.argv[1:]
print("""\
%{
print(f'''\
%{{
_Pragma("GCC diagnostic ignored \\"-Wimplicit-fallthrough\\"")
#if __GNUC__ >= 15
_Pragma("GCC diagnostic ignored \\"-Wzero-as-null-pointer-constant\\"")
#endif
%}""")
print("""\
struct {}_name {{ const char* name; int id; }};
%}}
struct {name}_name {{ const char* name; int id; }};
%null-strings
%%""".format(name))
%%''')
for line in open(input):
line = line.rstrip()
s = line.replace('_', '-')
print("{}, {}{}".format(s, prefix, line))
print(f'{s}, {prefix}{line}')
src/shared/generate-syscall-list.py
+1 -1
View File
@@ -4,4 +4,4 @@
import sys
for line in open(sys.argv[1]):
print('"{}\\0"'.format(line.strip()))
print(f'"{line.strip()}\\0"')
src/test/generate-sym-test.py
+13 -13
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: LGPL-2.1-or-later
#
# ruff: noqa: E501 UP015
# ruff: noqa: UP015
import os
import re
@@ -120,43 +120,43 @@ def process_source_file(file: IO[str]) -> None:
continue
print("""/* SPDX-License-Identifier: LGPL-2.1-or-later */
print('''/* SPDX-License-Identifier: LGPL-2.1-or-later */
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
""")
''')
for header in sys.argv[3:]:
with open(header, 'r') as f:
if process_header_file(f):
print('#include "{}"'.format(header.split('/')[-1]))
print("""
print('''
/* We want to check deprecated symbols too, without complaining */
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
""")
''')
print("""
print('''
struct symbol {
const char *name;
const void *symbol;
};
static struct symbol symbols_from_sym[] = {""")
static struct symbol symbols_from_sym[] = {''')
with open(sys.argv[1], 'r') as f:
process_sym_file(f)
print(""" {}
}, symbols_from_header[] = {""")
print(''' {}
}, symbols_from_header[] = {''')
for header in sys.argv[3:]:
with open(header, 'r') as f:
print(process_header_file(f), end='')
print(""" {}
}, symbols_from_source[] = {""")
print(''' {}
}, symbols_from_source[] = {''')
for dirpath, _, filenames in sorted(os.walk(sys.argv[2])):
for filename in sorted(filenames):
@@ -168,7 +168,7 @@ for dirpath, _, filenames in sorted(os.walk(sys.argv[2])):
with p.open('rt') as f:
process_source_file(f)
print(""" {}
print(''' {}
};
static int sort_callback(const void *a, const void *b) {
@@ -239,4 +239,4 @@ int main(void) {
}
return n_error == 0 ? EXIT_SUCCESS : EXIT_FAILURE;
}""")
}''')
src/ukify/test/test_ukify.py
+319 -226
View File
@@ -33,8 +33,7 @@ except ImportError as e:
sys.exit(77)
try:
# pyflakes: noqa
import pefile # noqa
import pefile
except ImportError as e:
print(str(e), file=sys.stderr)
sys.exit(77)
@@ -59,42 +58,51 @@ except ValueError:
slow_tests = True
arg_tools = ['--tools', build_root] if build_root else []
if build_root and (
p := pathlib.Path(f"{build_root}/linux{ukify.guess_efi_arch()}.efi.stub")
).exists():
if (
build_root
and (p := pathlib.Path(f'{build_root}/linux{ukify.guess_efi_arch()}.efi.stub')).exists()
): # fmt: skip
arg_tools += ['--stub', p]
def systemd_measure():
opts = ukify.create_parser().parse_args(arg_tools)
return ukify.find_tool('systemd-measure', opts=opts)
def test_guess_efi_arch():
arch = ukify.guess_efi_arch()
assert arch in ukify.EFI_ARCHES
def test_shell_join():
assert ukify.shell_join(['a', 'b', ' ']) == "a b ' '"
def test_round_up():
assert ukify.round_up(0) == 0
assert ukify.round_up(4095) == 4096
assert ukify.round_up(4096) == 4096
assert ukify.round_up(4097) == 8192
def test_namespace_creation():
ns = ukify.create_parser().parse_args(())
assert ns.linux is None
assert ns.initrd is None
def test_config_example():
ex = ukify.config_example()
assert '[UKI]' in ex
assert 'Splash = BMP' in ex
def test_apply_config(tmp_path):
config = tmp_path / 'config1.conf'
config.write_text(textwrap.dedent(
f'''
config.write_text(
textwrap.dedent(
f'''
[UKI]
Linux = LINUX
Initrd = initrd1 initrd2
@@ -117,7 +125,9 @@ def test_apply_config(tmp_path):
PCRPrivateKey = some/path7
PCRPublicKey = some/path8
Phases = {':'.join(ukify.KNOWN_PHASES)}
'''))
'''
)
)
ns = ukify.create_parser().parse_args(['build'])
ns.linux = None
@@ -125,9 +135,11 @@ def test_apply_config(tmp_path):
ukify.apply_config(ns, config)
assert ns.linux == pathlib.Path('LINUX')
assert ns.initrd == [pathlib.Path('initrd1'),
pathlib.Path('initrd2'),
pathlib.Path('initrd3')]
assert ns.initrd == [
pathlib.Path('initrd1'),
pathlib.Path('initrd2'),
pathlib.Path('initrd3'),
]
assert ns.cmdline == '1 2 3 4 5\n6 7 8'
assert ns.os_release == '@some/path1'
assert ns.devicetree == pathlib.Path('some/path2')
@@ -148,9 +160,11 @@ def test_apply_config(tmp_path):
ukify.finalize_options(ns)
assert ns.linux == pathlib.Path('LINUX')
assert ns.initrd == [pathlib.Path('initrd1'),
pathlib.Path('initrd2'),
pathlib.Path('initrd3')]
assert ns.initrd == [
pathlib.Path('initrd1'),
pathlib.Path('initrd2'),
pathlib.Path('initrd3'),
]
assert ns.cmdline == '1 2 3 4 5 6 7 8'
assert ns.os_release == pathlib.Path('some/path1')
assert ns.devicetree == pathlib.Path('some/path2')
@@ -168,6 +182,7 @@ def test_apply_config(tmp_path):
assert ns.pcr_public_keys == ['some/path8']
assert ns.phase_path_groups == [['enter-initrd:leave-initrd:sysinit:ready:shutdown:final']]
def test_parse_args_minimal():
with pytest.raises(ValueError):
ukify.parse_args([])
@@ -175,32 +190,39 @@ def test_parse_args_minimal():
opts = ukify.parse_args('arg1 arg2'.split())
assert opts.linux == pathlib.Path('arg1')
assert opts.initrd == [pathlib.Path('arg2')]
assert opts.os_release in (pathlib.Path('/etc/os-release'),
pathlib.Path('/usr/lib/os-release'))
assert opts.os_release in (
pathlib.Path('/etc/os-release'),
pathlib.Path('/usr/lib/os-release'),
)
def test_parse_args_many_deprecated():
opts = ukify.parse_args(
['/ARG1', '///ARG2', '/ARG3 WITH SPACE',
'--cmdline=a b c',
'--os-release=K1=V1\nK2=V2',
'--devicetree=DDDDTTTT',
'--splash=splash',
'--pcrpkey=PATH',
'--uname=1.2.3',
'--stub=STUBPATH',
'--pcr-private-key=PKEY1',
'--pcr-public-key=PKEY2',
'--pcr-banks=SHA1,SHA256',
'--signing-engine=ENGINE',
'--secureboot-private-key=SBKEY',
'--secureboot-certificate=SBCERT',
'--sign-kernel',
'--no-sign-kernel',
'--tools=TOOLZ///',
'--output=OUTPUT',
'--measure',
'--no-measure',
])
[
'/ARG1',
'///ARG2',
'/ARG3 WITH SPACE',
'--cmdline=a b c',
'--os-release=K1=V1\nK2=V2',
'--devicetree=DDDDTTTT',
'--splash=splash',
'--pcrpkey=PATH',
'--uname=1.2.3',
'--stub=STUBPATH',
'--pcr-private-key=PKEY1',
'--pcr-public-key=PKEY2',
'--pcr-banks=SHA1,SHA256',
'--signing-engine=ENGINE',
'--secureboot-private-key=SBKEY',
'--secureboot-certificate=SBCERT',
'--sign-kernel',
'--no-sign-kernel',
'--tools=TOOLZ///',
'--output=OUTPUT',
'--measure',
'--no-measure',
]
)
assert opts.linux == pathlib.Path('/ARG1')
assert opts.initrd == [pathlib.Path('/ARG2'), pathlib.Path('/ARG3 WITH SPACE')]
assert opts.cmdline == 'a b c'
@@ -221,34 +243,37 @@ def test_parse_args_many_deprecated():
assert opts.output == pathlib.Path('OUTPUT')
assert opts.measure is False
def test_parse_args_many():
opts = ukify.parse_args(
['build',
'--linux=/ARG1',
'--initrd=///ARG2',
'--initrd=/ARG3 WITH SPACE',
'--cmdline=a b c',
'--os-release=K1=V1\nK2=V2',
'--devicetree=DDDDTTTT',
'--splash=splash',
'--pcrpkey=PATH',
'--uname=1.2.3',
'--stub=STUBPATH',
'--pcr-private-key=PKEY1',
'--pcr-public-key=PKEY2',
'--pcr-banks=SHA1,SHA256',
'--signing-engine=ENGINE',
'--secureboot-private-key=SBKEY',
'--secureboot-certificate=SBCERT',
'--sign-kernel',
'--no-sign-kernel',
'--tools=TOOLZ///',
'--output=OUTPUT',
'--measure',
'--no-measure',
'--policy-digest',
'--no-policy-digest',
])
[
'build',
'--linux=/ARG1',
'--initrd=///ARG2',
'--initrd=/ARG3 WITH SPACE',
'--cmdline=a b c',
'--os-release=K1=V1\nK2=V2',
'--devicetree=DDDDTTTT',
'--splash=splash',
'--pcrpkey=PATH',
'--uname=1.2.3',
'--stub=STUBPATH',
'--pcr-private-key=PKEY1',
'--pcr-public-key=PKEY2',
'--pcr-banks=SHA1,SHA256',
'--signing-engine=ENGINE',
'--secureboot-private-key=SBKEY',
'--secureboot-certificate=SBCERT',
'--sign-kernel',
'--no-sign-kernel',
'--tools=TOOLZ///',
'--output=OUTPUT',
'--measure',
'--no-measure',
'--policy-digest',
'--no-policy-digest',
]
)
assert opts.linux == pathlib.Path('/ARG1')
assert opts.initrd == [pathlib.Path('/ARG2'), pathlib.Path('/ARG3 WITH SPACE')]
assert opts.cmdline == 'a b c'
@@ -270,14 +295,17 @@ def test_parse_args_many():
assert opts.measure is False
assert opts.policy_digest is False
def test_parse_sections():
opts = ukify.parse_args(
['build',
'--linux=/ARG1',
'--initrd=/ARG2',
'--section=test:TESTTESTTEST',
'--section=test2:@FILE',
])
[
'build',
'--linux=/ARG1',
'--initrd=/ARG2',
'--section=test:TESTTESTTEST',
'--section=test2:@FILE',
]
)
assert opts.linux == pathlib.Path('/ARG1')
assert opts.initrd == [pathlib.Path('/ARG2')]
@@ -293,11 +321,13 @@ def test_parse_sections():
assert opts.sections[1].tmpfile is None
assert opts.sections[1].measure is False
def test_config_priority(tmp_path):
config = tmp_path / 'config1.conf'
# config: use pesign and give certdir + certname
config.write_text(textwrap.dedent(
f'''
config.write_text(
textwrap.dedent(
f'''
[UKI]
Linux = LINUX
Initrd = initrd1 initrd2
@@ -321,44 +351,50 @@ def test_config_priority(tmp_path):
PCRPrivateKey = some/path7
PCRPublicKey = some/path8
Phases = {':'.join(ukify.KNOWN_PHASES)}
'''))
'''
)
)
# args: use sbsign and give key + cert, should override pesign
opts = ukify.parse_args(
['build',
'--linux=/ARG1',
'--initrd=///ARG2',
'--initrd=/ARG3 WITH SPACE',
'--cmdline= a b c ',
'--os-release=K1=V1\nK2=V2',
'--devicetree=DDDDTTTT',
'--splash=splash',
'--pcrpkey=PATH',
'--uname=1.2.3',
'--stub=STUBPATH',
'--pcr-private-key=PKEY1',
'--pcr-public-key=PKEY2',
'--pcr-banks=SHA1,SHA256',
'--signing-engine=ENGINE',
'--signtool=sbsign',
'--secureboot-private-key=SBKEY',
'--secureboot-certificate=SBCERT',
'--sign-kernel',
'--no-sign-kernel',
'--tools=TOOLZ///',
'--output=OUTPUT',
'--measure',
])
[
'build',
'--linux=/ARG1',
'--initrd=///ARG2',
'--initrd=/ARG3 WITH SPACE',
'--cmdline= a b c ',
'--os-release=K1=V1\nK2=V2',
'--devicetree=DDDDTTTT',
'--splash=splash',
'--pcrpkey=PATH',
'--uname=1.2.3',
'--stub=STUBPATH',
'--pcr-private-key=PKEY1',
'--pcr-public-key=PKEY2',
'--pcr-banks=SHA1,SHA256',
'--signing-engine=ENGINE',
'--signtool=sbsign',
'--secureboot-private-key=SBKEY',
'--secureboot-certificate=SBCERT',
'--sign-kernel',
'--no-sign-kernel',
'--tools=TOOLZ///',
'--output=OUTPUT',
'--measure',
]
)
ukify.apply_config(opts, config)
ukify.finalize_options(opts)
assert opts.linux == pathlib.Path('/ARG1')
assert opts.initrd == [pathlib.Path('initrd1'),
pathlib.Path('initrd2'),
pathlib.Path('initrd3'),
pathlib.Path('/ARG2'),
pathlib.Path('/ARG3 WITH SPACE')]
assert opts.initrd == [
pathlib.Path('initrd1'),
pathlib.Path('initrd2'),
pathlib.Path('initrd3'),
pathlib.Path('/ARG2'),
pathlib.Path('/ARG3 WITH SPACE'),
]
assert opts.cmdline == 'a b c'
assert opts.os_release == 'K1=V1\nK2=V2'
assert opts.devicetree == pathlib.Path('DDDDTTTT')
@@ -370,16 +406,17 @@ def test_config_priority(tmp_path):
assert opts.pcr_public_keys == ['PKEY2', 'some/path8']
assert opts.pcr_banks == ['SHA1', 'SHA256']
assert opts.signing_engine == 'ENGINE'
assert opts.signtool == 'sbsign' # from args
assert opts.sb_key == 'SBKEY' # from args
assert opts.sb_cert == pathlib.Path('SBCERT') # from args
assert opts.sb_certdir == 'some/path5' # from config
assert opts.sb_cert_name == 'some/name1' # from config
assert opts.signtool == 'sbsign' # from args
assert opts.sb_key == 'SBKEY' # from args
assert opts.sb_cert == pathlib.Path('SBCERT') # from args
assert opts.sb_certdir == 'some/path5' # from config
assert opts.sb_cert_name == 'some/name1' # from config
assert opts.sign_kernel is False
assert opts.tools == [pathlib.Path('TOOLZ/')]
assert opts.output == pathlib.Path('OUTPUT')
assert opts.measure is True
def test_help(capsys):
with pytest.raises(SystemExit):
ukify.parse_args(['--help'])
@@ -387,6 +424,7 @@ def test_help(capsys):
assert '--section' in out.out
assert not out.err
def test_help_display(capsys):
with pytest.raises(SystemExit):
ukify.parse_args(['inspect', '--help'])
@@ -394,6 +432,7 @@ def test_help_display(capsys):
assert '--section' in out.out
assert not out.err
def test_help_error_deprecated(capsys):
with pytest.raises(SystemExit):
ukify.parse_args(['a', 'b', '--no-such-option'])
@@ -402,6 +441,7 @@ def test_help_error_deprecated(capsys):
assert '--no-such-option' in out.err
assert len(out.err.splitlines()) == 1
def test_help_error(capsys):
with pytest.raises(SystemExit):
ukify.parse_args(['build', '--no-such-option'])
@@ -410,6 +450,7 @@ def test_help_error(capsys):
assert '--no-such-option' in out.err
assert len(out.err.splitlines()) == 1
@pytest.fixture(scope='session')
def kernel_initrd():
items = sorted(glob.glob('/lib/modules/*/vmlinuz'))
@@ -426,6 +467,7 @@ def kernel_initrd():
# We don't look _into_ the initrd. Any file is OK.
return ['--linux', linux, '--initrd', ukify.__file__]
def test_check_splash():
try:
# pyflakes: noqa
@@ -436,16 +478,19 @@ def test_check_splash():
with pytest.raises(OSError):
ukify.check_splash(os.devnull)
def test_basic_operation(kernel_initrd, tmp_path):
if kernel_initrd is None:
pytest.skip('linux+initrd not found')
output = f'{tmp_path}/basic.efi'
opts = ukify.parse_args([
'build',
*kernel_initrd,
f'--output={output}',
])
opts = ukify.parse_args(
[
'build',
*kernel_initrd,
f'--output={output}',
]
)
try:
ukify.check_inputs(opts)
except OSError as e:
@@ -458,20 +503,23 @@ def test_basic_operation(kernel_initrd, tmp_path):
shutil.rmtree(tmp_path)
def test_sections(kernel_initrd, tmp_path):
if kernel_initrd is None:
pytest.skip('linux+initrd not found')
output = f'{tmp_path}/basic.efi'
opts = ukify.parse_args([
'build',
*kernel_initrd,
f'--output={output}',
'--uname=1.2.3',
'--cmdline=ARG1 ARG2 ARG3',
'--os-release=K1=V1\nK2=V2\n',
'--section=.test:CONTENTZ',
])
opts = ukify.parse_args(
[
'build',
*kernel_initrd,
f'--output={output}',
'--uname=1.2.3',
'--cmdline=ARG1 ARG2 ARG3',
'--os-release=K1=V1\nK2=V2\n',
'--section=.test:CONTENTZ',
]
)
try:
ukify.check_inputs(opts)
@@ -484,24 +532,25 @@ def test_sections(kernel_initrd, tmp_path):
dump = subprocess.check_output(['objdump', '-h', output], text=True)
for sect in 'text osrel cmdline linux initrd uname test'.split():
assert re.search(fr'^\s*\d+\s+\.{sect}\s+[0-9a-f]+', dump, re.MULTILINE)
assert re.search(rf'^\s*\d+\s+\.{sect}\s+[0-9a-f]+', dump, re.MULTILINE)
shutil.rmtree(tmp_path)
def test_addon(tmp_path):
output = f'{tmp_path}/addon.efi'
args = [
'build',
f'--output={output}',
'--cmdline=ARG1 ARG2 ARG3',
"""--sbat=sbat,1,foo
'''--sbat=sbat,1,foo
foo,1
bar,2
""",
''',
'--section=.test:CONTENTZ',
"""--sbat=sbat,1,foo
'''--sbat=sbat,1,foo
baz,3
"""
''',
]
if stub := os.getenv('EFI_ADDON'):
args += [f'--stub={stub}']
@@ -521,26 +570,33 @@ baz,3
dump = subprocess.check_output(['objdump', '-h', output], text=True)
for sect in 'text cmdline test sbat'.split():
assert re.search(fr'^\s*\d+\s+\.{sect}\s+[0-9a-f]+', dump, re.MULTILINE)
assert re.search(rf'^\s*\d+\s+\.{sect}\s+[0-9a-f]+', dump, re.MULTILINE)
pe = pefile.PE(output, fast_load=True)
found = False
for section in pe.sections:
if section.Name.rstrip(b"\x00").decode() == ".sbat":
if section.Name.rstrip(b'\x00').decode() == '.sbat':
assert found is False
split = section.get_data().rstrip(b"\x00").decode().splitlines()
assert split == ["sbat,1,SBAT Version,sbat,1,https://github.com/rhboot/shim/blob/main/SBAT.md", "foo,1", "bar,2", "baz,3"]
split = section.get_data().rstrip(b'\x00').decode().splitlines()
assert split == [
'sbat,1,SBAT Version,sbat,1,https://github.com/rhboot/shim/blob/main/SBAT.md',
'foo,1',
'bar,2',
'baz,3',
]
found = True
assert found is True
def unbase64(filename):
tmp = tempfile.NamedTemporaryFile()
base64.decode(filename.open('rb'), tmp)
tmp.flush()
return tmp
def test_uname_scraping(kernel_initrd):
if kernel_initrd is None:
pytest.skip('linux+initrd not found')
@@ -549,8 +605,9 @@ def test_uname_scraping(kernel_initrd):
uname = ukify.Uname.scrape(kernel_initrd[1])
assert re.match(r'\d+\.\d+\.\d+', uname)
@pytest.mark.skipif(not slow_tests, reason='slow')
@pytest.mark.parametrize("days", [365*10, None])
@pytest.mark.parametrize('days', [365 * 10, None])
def test_efi_signing_sbsign(days, kernel_initrd, tmp_path):
if kernel_initrd is None:
pytest.skip('linux+initrd not found')
@@ -585,16 +642,20 @@ def test_efi_signing_sbsign(days, kernel_initrd, tmp_path):
if shutil.which('sbverify'):
# let's check that sbverify likes the resulting file
dump = subprocess.check_output([
'sbverify',
'--cert', cert.name,
output,
], text=True)
dump = subprocess.check_output(
[
'sbverify',
'--cert', cert.name,
output,
],
text=True,
) # fmt: skip
assert 'Signature verification OK' in dump
shutil.rmtree(tmp_path)
@pytest.mark.skipif(not slow_tests, reason='slow')
def test_efi_signing_pesign(kernel_initrd, tmp_path):
if kernel_initrd is None:
@@ -613,16 +674,18 @@ def test_efi_signing_pesign(kernel_initrd, tmp_path):
subprocess.check_call(cmd)
output = f'{tmp_path}/signed.efi'
opts = ukify.parse_args([
'build',
*kernel_initrd,
f'--output={output}',
'--uname=1.2.3',
'--signtool=pesign',
'--cmdline=ARG1 ARG2 ARG3',
f'--secureboot-certificate-name={name}',
f'--secureboot-certificate-dir={nss_db}',
])
opts = ukify.parse_args(
[
'build',
*kernel_initrd,
f'--output={output}',
'--uname=1.2.3',
'--signtool=pesign',
'--cmdline=ARG1 ARG2 ARG3',
f'--secureboot-certificate-name={name}',
f'--secureboot-certificate-dir={nss_db}',
]
)
try:
ukify.check_inputs(opts)
@@ -632,15 +695,20 @@ def test_efi_signing_pesign(kernel_initrd, tmp_path):
ukify.make_uki(opts)
# let's check that pesign likes the resulting file
dump = subprocess.check_output([
'pesign', '-S',
'-i', output,
], text=True)
dump = subprocess.check_output(
[
'pesign',
'-S',
'-i', output,
],
text=True,
) # fmt: skip
assert f"The signer's common name is {author}" in dump
shutil.rmtree(tmp_path)
def test_inspect(kernel_initrd, tmp_path, capsys, osrel=True):
if kernel_initrd is None:
pytest.skip('linux+initrd not found')
@@ -652,9 +720,9 @@ def test_inspect(kernel_initrd, tmp_path, capsys, osrel=True):
key = unbase64(ourdir / 'example.signing.key.base64')
output = f'{tmp_path}/signed2.efi'
uname_arg='1.2.3'
osrel_arg='Linux' if osrel else ''
cmdline_arg='ARG1 ARG2 ARG3'
uname_arg = '1.2.3'
osrel_arg = 'Linux' if osrel else ''
cmdline_arg = 'ARG1 ARG2 ARG3'
args = [
'build',
@@ -698,9 +766,11 @@ def test_inspect(kernel_initrd, tmp_path, capsys, osrel=True):
shutil.rmtree(tmp_path)
def test_inspect_no_osrel(kernel_initrd, tmp_path, capsys):
test_inspect(kernel_initrd, tmp_path, capsys, osrel=False)
@pytest.mark.skipif(not slow_tests, reason='slow')
def test_pcr_signing(kernel_initrd, tmp_path):
if kernel_initrd is None:
@@ -722,7 +792,7 @@ def test_pcr_signing(kernel_initrd, tmp_path):
'--uname=1.2.3',
'--cmdline=ARG1 ARG2 ARG3',
'--os-release=ID=foobar\n',
'--pcr-banks=sha384', # sha1 might not be allowed, use something else
'--pcr-banks=sha384', # sha1 might not be allowed, use something else
f'--pcr-private-key={priv.name}',
] + arg_tools
@@ -743,21 +813,25 @@ def test_pcr_signing(kernel_initrd, tmp_path):
dump = subprocess.check_output(['objdump', '-h', output], text=True)
for sect in 'text osrel cmdline linux initrd uname pcrsig'.split():
assert re.search(fr'^\s*\d+\s+\.{sect}\s+[0-9a-f]+', dump, re.MULTILINE)
assert re.search(rf'^\s*\d+\s+\.{sect}\s+[0-9a-f]+', dump, re.MULTILINE)
# objcopy fails when called without an output argument (EPERM).
# It also fails when called with /dev/null (file truncated).
# It also fails when called with /dev/zero (because it reads the
# output file, infinitely in this case.)
# So let's just call it with a dummy output argument.
subprocess.check_call([
'objcopy',
*(f'--dump-section=.{n}={tmp_path}/out.{n}' for n in (
'pcrpkey', 'pcrsig', 'osrel', 'uname', 'cmdline')),
output,
tmp_path / 'dummy',
],
text=True)
subprocess.check_call(
[
'objcopy',
*(
f'--dump-section=.{n}={tmp_path}/out.{n}'
for n in ('pcrpkey', 'pcrsig', 'osrel', 'uname', 'cmdline')
),
output,
tmp_path / 'dummy',
],
text=True,
)
assert open(tmp_path / 'out.pcrpkey').read() == open(pub.name).read()
assert open(tmp_path / 'out.osrel').read() == 'ID=foobar\n'
@@ -766,10 +840,11 @@ def test_pcr_signing(kernel_initrd, tmp_path):
sig = open(tmp_path / 'out.pcrsig').read()
sig = json.loads(sig)
assert list(sig.keys()) == ['sha384']
assert len(sig['sha384']) == 4 # four items for four phases
assert len(sig['sha384']) == 4 # four items for four phases
shutil.rmtree(tmp_path)
@pytest.mark.skipif(not slow_tests, reason='slow')
def test_pcr_signing2(kernel_initrd, tmp_path):
if kernel_initrd is None:
@@ -791,24 +866,27 @@ def test_pcr_signing2(kernel_initrd, tmp_path):
output = f'{tmp_path}/signed.efi'
assert kernel_initrd[0] == '--linux'
opts = ukify.parse_args([
'build',
*kernel_initrd[:2],
f'--initrd={microcode.name}',
*kernel_initrd[2:],
f'--output={output}',
'--uname=1.2.3',
'--cmdline=ARG1 ARG2 ARG3',
'--os-release=ID=foobar\n',
'--pcr-banks=sha384',
f'--pcrpkey={pub2.name}',
f'--pcr-public-key={pub.name}',
f'--pcr-private-key={priv.name}',
'--phases=enter-initrd enter-initrd:leave-initrd',
f'--pcr-public-key={pub2.name}',
f'--pcr-private-key={priv2.name}',
'--phases=sysinit ready shutdown final', # yes, those phase paths are not reachable
] + arg_tools)
opts = ukify.parse_args(
[
'build',
*kernel_initrd[:2],
f'--initrd={microcode.name}',
*kernel_initrd[2:],
f'--output={output}',
'--uname=1.2.3',
'--cmdline=ARG1 ARG2 ARG3',
'--os-release=ID=foobar\n',
'--pcr-banks=sha384',
f'--pcrpkey={pub2.name}',
f'--pcr-public-key={pub.name}',
f'--pcr-private-key={priv.name}',
'--phases=enter-initrd enter-initrd:leave-initrd',
f'--pcr-public-key={pub2.name}',
f'--pcr-private-key={priv2.name}',
'--phases=sysinit ready shutdown final', # yes, those phase paths are not reachable
]
+ arg_tools
)
try:
ukify.check_inputs(opts)
@@ -821,16 +899,20 @@ def test_pcr_signing2(kernel_initrd, tmp_path):
dump = subprocess.check_output(['objdump', '-h', output], text=True)
for sect in 'text osrel cmdline linux initrd uname pcrsig'.split():
assert re.search(fr'^\s*\d+\s+\.{sect}\s+[0-9a-f]+', dump, re.MULTILINE)
assert re.search(rf'^\s*\d+\s+\.{sect}\s+[0-9a-f]+', dump, re.MULTILINE)
subprocess.check_call([
'objcopy',
*(f'--dump-section=.{n}={tmp_path}/out.{n}' for n in (
'pcrpkey', 'pcrsig', 'osrel', 'uname', 'cmdline', 'initrd')),
output,
tmp_path / 'dummy',
],
text=True)
subprocess.check_call(
[
'objcopy',
*(
f'--dump-section=.{n}={tmp_path}/out.{n}'
for n in ('pcrpkey', 'pcrsig', 'osrel', 'uname', 'cmdline', 'initrd')
),
output,
tmp_path / 'dummy',
],
text=True,
)
assert open(tmp_path / 'out.pcrpkey').read() == open(pub2.name).read()
assert open(tmp_path / 'out.osrel').read() == 'ID=foobar\n'
@@ -841,22 +923,25 @@ def test_pcr_signing2(kernel_initrd, tmp_path):
sig = open(tmp_path / 'out.pcrsig').read()
sig = json.loads(sig)
assert list(sig.keys()) == ['sha384']
assert len(sig['sha384']) == 6 # six items for six phases paths
assert len(sig['sha384']) == 6 # six items for six phases paths
shutil.rmtree(tmp_path)
def test_key_cert_generation(tmp_path):
opts = ukify.parse_args([
'genkey',
f"--pcr-public-key={tmp_path / 'pcr1.pub.pem'}",
f"--pcr-private-key={tmp_path / 'pcr1.priv.pem'}",
'--phases=enter-initrd enter-initrd:leave-initrd',
f"--pcr-public-key={tmp_path / 'pcr2.pub.pem'}",
f"--pcr-private-key={tmp_path / 'pcr2.priv.pem'}",
'--phases=sysinit ready',
f"--secureboot-private-key={tmp_path / 'sb.priv.pem'}",
f"--secureboot-certificate={tmp_path / 'sb.cert.pem'}",
])
opts = ukify.parse_args(
[
'genkey',
f'--pcr-public-key={tmp_path / "pcr1.pub.pem"}',
f'--pcr-private-key={tmp_path / "pcr1.priv.pem"}',
'--phases=enter-initrd enter-initrd:leave-initrd',
f'--pcr-public-key={tmp_path / "pcr2.pub.pem"}',
f'--pcr-private-key={tmp_path / "pcr2.priv.pem"}',
'--phases=sysinit ready',
f'--secureboot-private-key={tmp_path / "sb.priv.pem"}',
f'--secureboot-certificate={tmp_path / "sb.cert.pem"}',
]
)
assert opts.verb == 'genkey'
ukify.check_cert_and_keys_nonexistent(opts)
@@ -867,39 +952,46 @@ def test_key_cert_generation(tmp_path):
if not shutil.which('openssl'):
return
for key in (tmp_path / 'pcr1.priv.pem',
tmp_path / 'pcr2.priv.pem',
tmp_path / 'sb.priv.pem'):
out = subprocess.check_output([
'openssl', 'rsa',
'-in', key,
'-text',
'-noout',
], text = True)
for key in (tmp_path / 'pcr1.priv.pem', tmp_path / 'pcr2.priv.pem', tmp_path / 'sb.priv.pem'):
out = subprocess.check_output(
[
'openssl', 'rsa',
'-in', key,
'-text',
'-noout',
],
text=True,
) # fmt: skip
assert 'Private-Key' in out
assert '2048 bit' in out
for pub in (tmp_path / 'pcr1.pub.pem',
tmp_path / 'pcr2.pub.pem'):
out = subprocess.check_output([
'openssl', 'rsa',
'-pubin',
'-in', pub,
'-text',
'-noout',
], text = True)
for pub in (tmp_path / 'pcr1.pub.pem', tmp_path / 'pcr2.pub.pem'):
out = subprocess.check_output(
[
'openssl', 'rsa',
'-pubin',
'-in', pub,
'-text',
'-noout',
],
text=True,
) # fmt: skip
assert 'Public-Key' in out
assert '2048 bit' in out
out = subprocess.check_output([
'openssl', 'x509',
'-in', tmp_path / 'sb.cert.pem',
'-text',
'-noout',
], text = True)
out = subprocess.check_output(
[
'openssl', 'x509',
'-in', tmp_path / 'sb.cert.pem',
'-text',
'-noout',
],
text=True,
) # fmt: skip
assert 'Certificate' in out
assert re.search(r'Issuer: CN\s?=\s?SecureBoot signing key on host', out)
@pytest.mark.skipif(not slow_tests, reason='slow')
def test_join_pcrsig(capsys, kernel_initrd, tmp_path):
if kernel_initrd is None:
@@ -965,5 +1057,6 @@ def test_join_pcrsig(capsys, kernel_initrd, tmp_path):
shutil.rmtree(tmp_path)
if __name__ == '__main__':
sys.exit(pytest.main(sys.argv))
src/ukify/ukify.py
+17 -17
View File
@@ -599,7 +599,7 @@ class SystemdSbSign(SignTool):
)
cmd = [
tool,
"sign",
'sign',
'--private-key', opts.sb_key,
'--certificate', opts.sb_cert,
*(
@@ -1109,7 +1109,7 @@ def pe_add_sections(opts: UkifyConfig, uki: UKI, output: str) -> None:
encoded = json.dumps(j).encode()
if len(encoded) > section.SizeOfRawData:
raise PEError(
f'Not enough space in existing section .pcrsig of size {section.SizeOfRawData} to append new data of size {len(encoded)}' # noqa: E501
f'Not enough space in existing section .pcrsig of size {section.SizeOfRawData} to append new data of size {len(encoded)}'
)
section.Misc_VirtualSize = len(encoded)
# bytes(n) results in an array of n zeroes
@@ -1305,15 +1305,15 @@ def parse_efifw_dir(path: Path) -> bytes:
return efifw_blob
STUB_SBAT = """\
STUB_SBAT = '''\
sbat,1,SBAT Version,sbat,1,https://github.com/rhboot/shim/blob/main/SBAT.md
uki,1,UKI,uki,1,https://uapi-group.org/specifications/specs/unified_kernel_image/
"""
'''
ADDON_SBAT = """\
ADDON_SBAT = '''\
sbat,1,SBAT Version,sbat,1,https://github.com/rhboot/shim/blob/main/SBAT.md
uki-addon,1,UKI Addon,addon,1,https://www.freedesktop.org/software/systemd/man/latest/systemd-stub.html
"""
'''
def make_uki(opts: UkifyConfig) -> None:
@@ -1498,7 +1498,7 @@ def make_uki(opts: UkifyConfig) -> None:
if names[0] != '.profile':
raise ValueError(
f'Expected .profile section as first valid section in PE profile binary {profile} but got {names[0]}' # noqa: E501
f'Expected .profile section as first valid section in PE profile binary {profile} but got {names[0]}'
)
if names.count('.profile') > 1:
@@ -1688,7 +1688,7 @@ def generate_keys(opts: UkifyConfig) -> None:
if not work:
raise ValueError(
'genkey: --secureboot-private-key=/--secureboot-certificate= or --pcr-private-key/--pcr-public-key must be specified' # noqa: E501
'genkey: --secureboot-private-key=/--secureboot-certificate= or --pcr-private-key/--pcr-public-key must be specified'
)
@@ -1879,7 +1879,7 @@ class ConfigItem:
elif self.type:
conv = self.type
else:
conv = lambda s: s # noqa: E731
conv = lambda s: s
# This is a bit ugly, but --initrd and --devicetree-auto are the only options
# with multiple args on the command line and a space-separated list in the
@@ -2116,14 +2116,14 @@ CONFIG_ITEMS = [
ConfigItem(
'--secureboot-private-key',
dest='sb_key',
help='required by --signtool=sbsign|systemd-sbsign. Path to key file or engine/provider designation for SB signing', # noqa: E501
help='required by --signtool=sbsign|systemd-sbsign. Path to key file or engine/provider designation for SB signing',
config_key='UKI/SecureBootPrivateKey',
),
ConfigItem(
'--secureboot-certificate',
dest='sb_cert',
help=(
'required by --signtool=sbsign. sbsign needs a path to certificate file or engine-specific designation for SB signing' # noqa: E501
'required by --signtool=sbsign. sbsign needs a path to certificate file or engine-specific designation for SB signing'
),
config_key='UKI/SecureBootCertificate',
),
@@ -2132,7 +2132,7 @@ CONFIG_ITEMS = [
dest='sb_certdir',
default='/etc/pki/pesign',
help=(
'required by --signtool=pesign. Path to nss certificate database directory for PE signing. Default is /etc/pki/pesign' # noqa: E501
'required by --signtool=pesign. Path to nss certificate database directory for PE signing. Default is /etc/pki/pesign'
),
config_key='UKI/SecureBootCertificateDir',
config_push=ConfigItem.config_set,
@@ -2141,7 +2141,7 @@ CONFIG_ITEMS = [
'--secureboot-certificate-name',
dest='sb_cert_name',
help=(
'required by --signtool=pesign. pesign needs a certificate nickname of nss certificate database entry to use for PE signing' # noqa: E501
'required by --signtool=pesign. pesign needs a certificate nickname of nss certificate database entry to use for PE signing'
),
config_key='UKI/SecureBootCertificateName',
),
@@ -2326,11 +2326,11 @@ def create_parser() -> argparse.ArgumentParser:
p = argparse.ArgumentParser(
description='Build and sign Unified Kernel Images',
usage='\n '
+ textwrap.dedent("""\
+ textwrap.dedent('''\
ukify {b}build{e} [--linux=LINUX] [--initrd=INITRD] [options…]
ukify {b}genkey{e} [options…]
ukify {b}inspect{e} FILE… [options…]
""").format(b=Style.bold, e=Style.reset),
''').format(b=Style.bold, e=Style.reset),
allow_abbrev=False,
add_help=False,
epilog='\n '.join(('config file:', *config_example())),
@@ -2458,7 +2458,7 @@ def finalize_options(opts: argparse.Namespace) -> None:
# both param given, infer sbsign and in case it was given, ensure signtool=sbsign
if opts.signtool and opts.signtool not in ('sbsign', 'systemd-sbsign'):
raise ValueError(
f'Cannot provide --signtool={opts.signtool} with --secureboot-private-key= and --secureboot-certificate=' # noqa: E501
f'Cannot provide --signtool={opts.signtool} with --secureboot-private-key= and --secureboot-certificate='
)
if not opts.signtool:
opts.signtool = 'sbsign'
@@ -2478,7 +2478,7 @@ def finalize_options(opts: argparse.Namespace) -> None:
if opts.sign_kernel and not opts.sb_key and not opts.sb_cert_name:
raise ValueError(
'--sign-kernel requires either --secureboot-private-key= and --secureboot-certificate= (for sbsign) or --secureboot-certificate-name= (for pesign) to be specified' # noqa: E501
'--sign-kernel requires either --secureboot-private-key= and --secureboot-certificate= (for sbsign) or --secureboot-certificate-name= (for pesign) to be specified'
)
opts.profile = resolve_at_path(opts.profile)
test/create-sys-script.py
+30 -29
View File
@@ -1,28 +1,29 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: LGPL-2.1-or-later
OUTFILE_HEADER = """#!/usr/bin/env python3
import filecmp
import os
import stat
import subprocess
import sys
import tempfile
OUTFILE_HEADER = '''#!/usr/bin/env python3
# SPDX-License-Identifier: LGPL-2.1-or-later
#
# create-sys-script.py
#
# © 2017 Canonical Ltd.
# Author: Dan Streetman <dan.streetman@canonical.com>
"""
'''
# Use this only to (re-)create the test/sys-script.py script,
# after adding or modifying anything in the test/sys/ directory
import os, sys
import stat
import tempfile
import filecmp
import subprocess
OUTFILE_MODE = 0o775
OUTFILE_FUNCS = r"""
OUTFILE_FUNCS = r'''
import os, sys
import shutil
@@ -36,9 +37,9 @@ def f(path, mode, contents):
with open(path, "wb") as f:
f.write(contents)
os.chmod(path, mode)
"""
'''
OUTFILE_MAIN = """
OUTFILE_MAIN = '''
if len(sys.argv) < 2:
exit("Usage: {} <target dir>".format(sys.argv[0]))
@@ -49,7 +50,7 @@ os.chdir(sys.argv[1])
if os.path.exists('sys'):
shutil.rmtree('sys')
"""
'''
def handle_dir(outfile, path):
@@ -67,17 +68,17 @@ def escape_single_quotes(b):
r = repr(b)[2:-1]
# python escapes all ' only if there are ' and " in the string
if '"' not in r:
r = r.replace("'", r"\'")
r = r.replace("'", r'\'')
# return line with all ' escaped
return r
def handle_file(outfile, path):
m = os.lstat(path).st_mode & 0o777
with open(path, "rb") as f:
with open(path, 'rb') as f:
b = f.read()
if b.count(b"\n") > 1:
r = "\n".join( escape_single_quotes(l) for l in b.split(b"\n") )
if b.count(b'\n') > 1:
r = '\n'.join(escape_single_quotes(line) for line in b.split(b'\n'))
r = f"b'''{r}'''"
else:
r = repr(b)
@@ -85,7 +86,7 @@ def handle_file(outfile, path):
def process_sysdir(outfile):
for (dirpath, dirnames, filenames) in os.walk('sys'):
for dirpath, dirnames, filenames in os.walk('sys'):
handle_dir(outfile, dirpath)
for d in dirnames:
path = os.path.join(dirpath, d)
@@ -105,17 +106,17 @@ def verify_dir(tmpd, path_a):
mode_a = os.lstat(path_a).st_mode
mode_b = os.lstat(path_b).st_mode
if not stat.S_ISDIR(mode_b):
raise Exception("Not directory")
raise Exception('Not directory')
if (mode_a & 0o777) != (mode_b & 0o777):
raise Exception("Permissions mismatch")
raise Exception('Permissions mismatch')
def verify_link(tmpd, path_a):
path_b = os.path.join(tmpd, path_a)
if not stat.S_ISLNK(os.lstat(path_b).st_mode):
raise Exception("Not symlink")
raise Exception('Not symlink')
if os.readlink(path_a) != os.readlink(path_b):
raise Exception("Symlink dest mismatch")
raise Exception('Symlink dest mismatch')
def verify_file(tmpd, path_a):
@@ -123,16 +124,16 @@ def verify_file(tmpd, path_a):
mode_a = os.lstat(path_a).st_mode
mode_b = os.lstat(path_b).st_mode
if not stat.S_ISREG(mode_b):
raise Exception("Not file")
raise Exception('Not file')
if (mode_a & 0o777) != (mode_b & 0o777):
raise Exception("Permissions mismatch")
raise Exception('Permissions mismatch')
if not filecmp.cmp(path_a, path_b, shallow=False):
raise Exception("File contents mismatch")
raise Exception('File contents mismatch')
def verify_script(tmpd):
any = False
for (dirpath, dirnames, filenames) in os.walk("sys"):
for dirpath, dirnames, filenames in os.walk('sys'):
any = True
try:
path = dirpath
@@ -154,7 +155,8 @@ def verify_script(tmpd):
if not any:
exit('Nothing found!')
if __name__ == "__main__":
if __name__ == '__main__':
if len(sys.argv) < 2:
exit('Usage: create-sys-script.py /path/to/test/')
@@ -163,10 +165,9 @@ if __name__ == "__main__":
os.chdir(sys.argv[1])
with open(outfile, "w") as f:
with open(outfile, 'w') as f:
os.chmod(outfile, OUTFILE_MODE)
f.write(OUTFILE_HEADER.replace(os.path.basename(sys.argv[0]),
os.path.basename(outfile)))
f.write(OUTFILE_HEADER.replace(os.path.basename(sys.argv[0]), os.path.basename(outfile)))
f.write(OUTFILE_FUNCS)
f.write(OUTFILE_MAIN)
process_sysdir(f)
test/fuzz/generate-directives.py
+2 -1
View File
@@ -1,8 +1,9 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: LGPL-2.1-or-later
import collections
import re
import sys
import collections, re
d = collections.defaultdict(list)
for line in open(sys.argv[1]):
test/integration-tests/TEST-04-JOURNAL/TEST-04-JOURNAL.units/logs-filtering-syslog.py
+3 -6
View File
@@ -5,10 +5,7 @@ import subprocess
import syslog
if __name__ == '__main__':
syslog.openlog(ident="logs-filtering", logoption=syslog.LOG_PID)
syslog.syslog(syslog.LOG_NOTICE, "Logging from the service, and ~more~ foo bar")
syslog.openlog(ident='logs-filtering', logoption=syslog.LOG_PID)
syslog.syslog(syslog.LOG_NOTICE, 'Logging from the service, and ~more~ foo bar')
subprocess.check_output(
['journalctl', '--sync'],
stdin=subprocess.DEVNULL,
text=True)
subprocess.check_output(['journalctl', '--sync'], stdin=subprocess.DEVNULL, text=True)
test/integration-tests/TEST-74-AUX-UTILS/TEST-74-AUX-UTILS.units/fake-imds.py
+22 -18
View File
@@ -1,15 +1,17 @@
#!/usr/bin/python3
# SPDX-License-Identifier: LGPL-2.1-or-later
import os, socket
import os
import socket
from http.server import BaseHTTPRequestHandler, HTTPServer
def sd_notify(state: str) -> bool:
notify_socket = os.environ.get("NOTIFY_SOCKET")
notify_socket = os.environ.get('NOTIFY_SOCKET')
if not notify_socket:
return False
if notify_socket.startswith("@"):
notify_socket = "\0" + notify_socket[1:]
if notify_socket.startswith('@'):
notify_socket = '\0' + notify_socket[1:]
try:
with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
sock.sendto(state.encode(), notify_socket)
@@ -18,34 +20,36 @@ def sd_notify(state: str) -> bool:
return True
class Handler(BaseHTTPRequestHandler):
def do_GET(self):
if self.path == "/userdata":
body = b"{\"systemd.credentials\":[{\"name\":\"acredtest\",\"text\":\"avalue\"}]}"
if self.path == '/userdata':
body = b'{"systemd.credentials":[{"name":"acredtest","text":"avalue"}]}'
self.send_response(200)
self.send_header("Content-Type", "text/plain")
self.send_header("Content-Length", len(body))
self.send_header('Content-Type', 'text/plain')
self.send_header('Content-Length', len(body))
self.end_headers()
self.wfile.write(body)
elif self.path == "/hostname":
body = b"piff"
elif self.path == '/hostname':
body = b'piff'
self.send_response(200)
self.send_header("Content-Type", "text/plain")
self.send_header("Content-Length", len(body))
self.send_header('Content-Type', 'text/plain')
self.send_header('Content-Length', len(body))
self.end_headers()
self.wfile.write(body)
else:
self.send_error(404)
def log_message(self, fmt, *args):
print(f"{self.address_string()} - {fmt % args}")
print(f'{self.address_string()} - {fmt % args}')
PORT=8088
server = HTTPServer(("", PORT), Handler)
print(f"Serving on http://localhost:{PORT}/")
PORT = 8088
server = HTTPServer(('', PORT), Handler)
print(f'Serving on http://localhost:{PORT}/')
try:
sd_notify("READY=1")
sd_notify('READY=1')
server.serve_forever()
except KeyboardInterrupt:
print("\nStopped.")
print('\nStopped.')
test/integration-tests/TEST-74-AUX-UTILS/TEST-74-AUX-UTILS.units/fake-report-server.py
+28 -21
View File
@@ -1,15 +1,20 @@
#!/usr/bin/python3
# SPDX-License-Identifier: LGPL-2.1-or-later
import argparse, json, os, socket, ssl
import argparse
import json
import os
import socket
import ssl
from http.server import BaseHTTPRequestHandler, HTTPServer
def sd_notify(state: str) -> bool:
notify_socket = os.environ.get("NOTIFY_SOCKET")
notify_socket = os.environ.get('NOTIFY_SOCKET')
if not notify_socket:
return False
if notify_socket.startswith("@"):
notify_socket = "\0" + notify_socket[1:]
if notify_socket.startswith('@'):
notify_socket = '\0' + notify_socket[1:]
try:
with socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) as sock:
sock.sendto(state.encode(), notify_socket)
@@ -17,54 +22,56 @@ def sd_notify(state: str) -> bool:
return False
return True
class Handler(BaseHTTPRequestHandler):
def do_POST(self):
length = int(self.headers.get("Content-Length", 0))
length = int(self.headers.get('Content-Length', 0))
body = self.rfile.read(length)
# Check optional attribute
if auth := self.headers.get("Authorization"):
print(f"Authorization: {auth}")
if auth := self.headers.get('Authorization'):
print(f'Authorization: {auth}')
# Validate JSON structure
try:
data = json.loads(body)
except json.JSONDecodeError:
self.send_error(400, "Invalid JSON")
self.send_error(400, 'Invalid JSON')
return
print(f"JSON: {s if len(s := str(data)) < 80 else s[:40] + '…' + s[-40:]}")
if "metrics" not in data:
if 'metrics' not in data:
self.send_error(400, "Missing 'metrics' field")
return
response = json.dumps({"status": "ok"}).encode()
response = json.dumps({'status': 'ok'}).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", len(response))
self.send_header('Content-Type', 'application/json')
self.send_header('Content-Length', len(response))
self.end_headers()
self.wfile.write(response)
def log_message(self, fmt, *args):
print(f"{self.address_string()} - {fmt % args}")
print(f'{self.address_string()} - {fmt % args}')
parser = argparse.ArgumentParser()
parser.add_argument("--port", type=int, default=8089)
parser.add_argument("--cert", help="TLS certificate file")
parser.add_argument("--key", help="TLS private key file")
parser.add_argument('--port', type=int, default=8089)
parser.add_argument('--cert', help='TLS certificate file')
parser.add_argument('--key', help='TLS private key file')
args = parser.parse_args()
server = HTTPServer(("", args.port), Handler)
server = HTTPServer(('', args.port), Handler)
if args.cert and args.key:
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
ctx.load_cert_chain(args.cert, args.key)
server.socket = ctx.wrap_socket(server.socket, server_side=True)
print(f"Serving on https://localhost:{args.port}/")
print(f'Serving on https://localhost:{args.port}/')
else:
print(f"Serving on http://localhost:{args.port}/")
print(f'Serving on http://localhost:{args.port}/')
try:
sd_notify("READY=1")
sd_notify('READY=1')
server.serve_forever()
except KeyboardInterrupt:
print("\nStopped.")
print('\nStopped.')
test/integration-tests/TEST-74-AUX-UTILS/TEST-74-AUX-UTILS.units/proxy-echo.py
+2 -2
View File
@@ -5,10 +5,10 @@ import socket
import sys
data = sys.stdin.buffer.read()
s = socket.create_connection(("localhost", 12345), timeout=15)
s = socket.create_connection(('localhost', 12345), timeout=15)
s.settimeout(15)
s.sendall(data)
received = b""
received = b''
while len(received) < len(data):
chunk = s.recv(65536)
if not chunk:
test/integration-tests/integration-test-wrapper.py
+22 -22
View File
@@ -21,13 +21,13 @@ from pathlib import Path
from types import FrameType
from typing import Optional
EMERGENCY_EXIT_DROPIN = """\
EMERGENCY_EXIT_DROPIN = '''\
[Unit]
Wants=emergency-exit.service
"""
'''
EMERGENCY_EXIT_SERVICE = """\
EMERGENCY_EXIT_SERVICE = '''\
[Unit]
DefaultDependencies=no
Conflicts=shutdown.target
@@ -38,7 +38,7 @@ FailureAction=exit
[Service]
ExecStart=false
"""
'''
@dataclasses.dataclass(frozen=True)
@@ -472,43 +472,43 @@ def main() -> None:
name = args.name + (f'-{i}' if (i := os.getenv('MESON_TEST_ITERATION')) else '')
dropin = textwrap.dedent(
"""\
'''\
[Service]
StandardOutput=journal+console
"""
'''
)
if not shell:
dropin += textwrap.dedent(
"""
'''
[Unit]
SuccessAction=exit
SuccessActionExitStatus=123
"""
'''
)
if os.getenv('TEST_MATCH_SUBTEST'):
dropin += textwrap.dedent(
f"""
f'''
[Service]
Environment=TEST_MATCH_SUBTEST={os.environ['TEST_MATCH_SUBTEST']}
"""
'''
)
if os.getenv('TEST_MATCH_TESTCASE'):
dropin += textwrap.dedent(
f"""
f'''
[Service]
Environment=TEST_MATCH_TESTCASE={os.environ['TEST_MATCH_TESTCASE']}
"""
'''
)
if os.getenv('TEST_RUN_DFUZZER'):
dropin += textwrap.dedent(
f"""
f'''
[Service]
Environment=TEST_RUN_DFUZZER={os.environ['TEST_RUN_DFUZZER']}
"""
'''
)
if os.getenv('TEST_JOURNAL_USE_TMP', '0') == '1':
@@ -525,14 +525,14 @@ def main() -> None:
if not sys.stdin.isatty():
dropin += textwrap.dedent(
"""
'''
[Unit]
FailureAction=exit
"""
'''
)
elif not shell:
dropin += textwrap.dedent(
"""
'''
[Unit]
Wants=multi-user.target getty-pre.target
Before=getty-pre.target
@@ -546,17 +546,17 @@ def main() -> None:
IgnoreSIGPIPE=no
# bash ignores SIGTERM
KillSignal=SIGHUP
"""
'''
)
if sys.stdin.isatty():
dropin += textwrap.dedent(
"""
'''
[Service]
ExecStartPre=/usr/lib/systemd/tests/testdata/integration-test-setup.sh setup
ExecStopPost=/usr/lib/systemd/tests/testdata/integration-test-setup.sh finalize
StateDirectory=%N
"""
'''
)
if args.rtc:
@@ -589,7 +589,7 @@ def main() -> None:
*(['--forward-journal', journal_file] if journal_file else []),
*(
[
'--credential', f'systemd.extra-unit.emergency-exit.service={shlex.quote(EMERGENCY_EXIT_SERVICE)}', # noqa: E501
'--credential', f'systemd.extra-unit.emergency-exit.service={shlex.quote(EMERGENCY_EXIT_SERVICE)}',
'--credential', f'systemd.unit-dropin.emergency.target={shlex.quote(EMERGENCY_EXIT_DROPIN)}',
]
if not sys.stdin.isatty()
@@ -700,7 +700,7 @@ def main() -> None:
iter = os.environ['GITHUB_RUN_ATTEMPT']
runner = os.environ['TEST_RUNNER']
artifact = (
f'ci-{wf}-{id}-{iter}-{summary.distribution}-{summary.release}-{runner}-failed-test-journals' # noqa: E501
f'ci-{wf}-{id}-{iter}-{summary.distribution}-{summary.release}-{runner}-failed-test-journals'
)
ops += [f'gh run download {id} --name {artifact} -D ci/{artifact}']
journal_file = Path(f'ci/{artifact}/test/journal/{name}.journal')
test/networkd-test.py
+343 -218
View File
File diff suppressed because it is too large Load Diff
test/rule-syntax-check.py
+14 -11
View File
@@ -14,21 +14,21 @@ if not rules_files:
sys.exit('Specify files to test as arguments')
quoted_string_re = r'"(?:[^\\"]|\\.)*"'
no_args_tests = re.compile(r'(ACTION|DEVPATH|KERNELS?|NAME|SYMLINK|SUBSYSTEMS?|DRIVERS?|TAG|PROGRAM|RESULT|TEST)\s*(?:=|!)=\s*' + quoted_string_re + '$')
no_args_tests = re.compile(rf'(ACTION|DEVPATH|KERNELS?|NAME|SYMLINK|SUBSYSTEMS?|DRIVERS?|TAG|PROGRAM|RESULT|TEST)\s*(?:=|!)=\s*{quoted_string_re}$') # fmt: skip
# PROGRAM can also be specified as an assignment.
program_assign = re.compile(r'PROGRAM\s*=\s*' + quoted_string_re + '$')
args_tests = re.compile(r'(ATTRS?|ENV|CONST|TEST){([a-zA-Z0-9/_.*%-]+)}\s*(?:=|!)=\s*' + quoted_string_re + '$')
no_args_assign = re.compile(r'(NAME|SYMLINK|OWNER|GROUP|MODE|TAG|RUN|LABEL|GOTO|OPTIONS|IMPORT)\s*(?:\+=|:=|=)\s*' + quoted_string_re + '$')
args_assign = re.compile(r'(ATTR|ENV|IMPORT|RUN){([a-zA-Z0-9/_.*%-]+)}\s*(=|\+=)\s*' + quoted_string_re + '$')
args_tests = re.compile(rf'(ATTRS?|ENV|CONST|TEST){{([a-zA-Z0-9/_.*%-]+)}}\s*(?:=|!)=\s*{quoted_string_re}$')
no_args_assign = re.compile(rf'(NAME|SYMLINK|OWNER|GROUP|MODE|TAG|RUN|LABEL|GOTO|OPTIONS|IMPORT)\s*(?:\+=|:=|=)\s*{quoted_string_re}$') # fmt: skip
args_assign = re.compile(rf'(ATTR|ENV|IMPORT|RUN){{([a-zA-Z0-9/_.*%-]+)}}\s*(=|\+=)\s*{quoted_string_re}$')
# Find comma-separated groups, but allow commas that are inside quoted strings.
# Using quoted_string_re + '?' so that strings missing the last double quote
# will still match for this part that splits on commas.
comma_separated_group_re = re.compile(r'(?:[^,"]|' + quoted_string_re + '?)+')
comma_separated_group_re = re.compile(rf'(?:[^,"]|{quoted_string_re}?)+')
result = 0
buffer = ''
for path in rules_files:
print('# looking at {}'.format(path))
print(f'# looking at {path}')
lineno = 0
for line in open(path):
lineno += 1
@@ -50,11 +50,14 @@ for path in rules_files:
# it generally improves the readability of the rules.
for clause_match in comma_separated_group_re.finditer(line):
clause = clause_match.group().strip()
if not (no_args_tests.match(clause) or args_tests.match(clause) or
no_args_assign.match(clause) or args_assign.match(clause) or
program_assign.match(clause)):
print('Invalid line {}:{}: {}'.format(path, lineno, line))
if not (
no_args_tests.match(clause)
or args_tests.match(clause)
or no_args_assign.match(clause)
or args_assign.match(clause)
or program_assign.match(clause)
):
print(f'Invalid line {path}:{lineno}: {line}')
print(' clause:', clause)
print()
result = 1
test/run-unit-tests.py
+23 -6
View File
@@ -6,8 +6,10 @@ import os
import pathlib
import subprocess
import sys
try:
import colorama as c
GREEN = c.Fore.GREEN
YELLOW = c.Fore.YELLOW
RED = c.Fore.RED
@@ -16,23 +18,38 @@ try:
except ImportError:
GREEN = YELLOW = RED = RESET_ALL = BRIGHT = ''
class total:
total = None
good = 0
skip = 0
fail = 0
def argument_parser():
p = argparse.ArgumentParser()
p.add_argument('-u', '--unsafe', action='store_true',
help='run "unsafe" tests too')
p.add_argument('-A', '--artifact_directory',
help='store output from failed tests in this dir')
p.add_argument('-s', '--skip', action='append', default=[],
help='skip the named test')
p.add_argument(
'-u',
'--unsafe',
action='store_true',
help='run "unsafe" tests too',
)
p.add_argument(
'-A',
'--artifact_directory',
help='store output from failed tests in this dir',
)
p.add_argument(
'-s',
'--skip',
action='append',
default=[],
help='skip the named test',
)
return p
opts = argument_parser().parse_args()
unittestdir = pathlib.Path(__file__).parent.absolute() / 'unit-tests'
test/sd-script.py
+55 -49
View File
@@ -1,6 +1,8 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: LGPL-2.1-or-later
#
# ruff: noqa: F821
#
# sd-script.py: create LOTS of sd device entries in fake sysfs
#
# (C) 2018 Martin Wilck, SUSE Linux GmbH
@@ -15,63 +17,67 @@
# Note that sys-script.py already creates 10 sd device nodes
# (sda+sdb and partitions). This script starts with sdc.
import re
import os
import errno
import os
import re
import sys
def d(path, mode):
os.mkdir(path, mode)
def l(path, src):
def l(path, src): # noqa: E743
os.symlink(src, path)
def f(path, mode, contents):
with open(path, "wb") as f:
with open(path, 'wb') as f:
f.write(contents)
os.chmod(path, mode)
class SD(object):
class SD:
sd_major = [8] + list(range(65, 72)) + list(range(128, 136))
_name_re = re.compile(r'sd(?P<f>[a-z]*)$')
def _init_from_name(self, name):
mt = self._name_re.match(name)
if mt is None:
raise RuntimeError("invalid name %s" % name)
nm = mt.group("f")
raise RuntimeError(f'invalid name {name}')
nm = mt.group('f')
base = 1
ls = nm[-1]
nm = nm[:-1]
n = base * (ord(ls)-ord('a'))
n = base * (ord(ls) - ord('a'))
while len(nm) > 0:
ls = nm[-1]
nm = nm[:-1]
base *= 26
n += base * (1 + ord(ls)-ord('a'))
n += base * (1 + ord(ls) - ord('a'))
self._num = n
def _init_from_dev(self, dev):
maj, min = dev.split(":")
maj, min = dev.split(':')
maj = self.sd_major.index(int(maj, 10))
min = int(min, 10)
num = int(min / 16)
self._num = 16*maj + num%16 + 256*int(num/16)
self._num = 16 * maj + num % 16 + 256 * int(num / 16)
@staticmethod
def _disk_num(a, b):
n = ord(a)-ord('a')
n = ord(a) - ord('a')
if b != '':
n = 26 * (n+1) + ord(b)-ord('a')
n = 26 * (n + 1) + ord(b) - ord('a')
return n
@staticmethod
def _get_major(n):
return SD.sd_major[(n%256)//16]
return SD.sd_major[(n % 256) // 16]
@staticmethod
def _get_minor(n):
return 16 * (n % 16 + 16 * n//256)
return 16 * (n % 16 + 16 * n // 256)
@staticmethod
def _get_name(n):
@@ -81,7 +87,7 @@ class SD(object):
while n >= 0:
s = chr(n % 26 + ord('a')) + s
n = n // 26 - 1
return "sd" + s
return 'sd' + s
@staticmethod
def _get_dev_t(n):
@@ -90,9 +96,9 @@ class SD(object):
return (maj << 20) + min
def __init__(self, arg):
if type(arg) is type(0):
if type(arg) is int:
self._num = arg
elif arg.startswith("sd"):
elif arg.startswith('sd'):
self._init_from_name(arg)
else:
self._init_from_dev(arg)
@@ -110,9 +116,7 @@ class SD(object):
return hash(self._num)
def __str__(self):
return "%s/%s" % (
self.devstr(),
self._get_name(self._num))
return f'{self.devstr()}/{self._get_name(self._num)}'
def major(self):
return self._get_major(self._num)
@@ -121,29 +125,27 @@ class SD(object):
return self._get_minor(self._num)
def devstr(self):
return "%d:%d" % (self._get_major(self._num),
self._get_minor(self._num))
return f'{self._get_major(self._num)}:{self._get_minor(self._num)}'
def namestr(self):
return self._get_name(self._num)
def longstr(self):
return "%d\t%s\t%s\t%08x" % (self._num,
self.devstr(),
self.namestr(),
self._get_dev_t(self._num))
return f'{self._num}\t{self.devstr()}\t{self.namestr()}\t{self._get_dev_t(self._num):08x}'
class MySD(SD):
def subst(self, first_sg):
disk = {
"lun": self._num,
"major": self.major(),
"devnode": self.namestr(),
"disk_minor": self.minor(),
"sg_minor": first_sg + self._num,
'lun': self._num,
'major': self.major(),
'devnode': self.namestr(),
'disk_minor': self.minor(),
'sg_minor': first_sg + self._num,
}
return disk
disk_template = r"""\
l('sys/bus/scsi/drivers/sd/7:0:0:{lun}', '../../../../devices/pci0000:00/0000:00:1d.7/usb5/5-1/5-1:1.0/host7/target7:0:0/7:0:0:{lun}')
l('sys/bus/scsi/devices/7:0:0:{lun}', '../../../devices/pci0000:00/0000:00:1d.7/usb5/5-1/5-1:1.0/host7/target7:0:0/7:0:0:{lun}')
@@ -282,29 +284,33 @@ DEVNAME={devnode}{part_num}
"""
if len(sys.argv) != 3:
exit("Usage: {} <target dir> <number>".format(sys.argv[0]))
exit(f'Usage: {sys.argv[0]} <target dir> <number>')
if not os.path.isdir(sys.argv[1]):
exit("Target dir {} not found".format(sys.argv[1]))
exit(f'Target dir {sys.argv[1]} not found')
def create_part_sysfs(disk, sd, prt):
part = disk
part.update ({
"part_num": prt,
"part_minor": disk["disk_minor"] + prt,
})
part.update(
{
'part_num': prt,
'part_minor': disk['disk_minor'] + prt,
}
)
try:
exec(part_template.format(**part))
except OSError:
si = sys.exc_info()[1]
if (si.errno == errno.EEXIST):
print("sysfs structures for %s%d exist" % (sd.namestr(), prt))
if si.errno == errno.EEXIST:
print(f'sysfs structures for {sd.namestr()}{prt} exist')
else:
print("error for %s%d: %s" % (sd.namestr(), prt, si[1]))
print(f'error for {sd.namestr()}{prt}: {si[1]}')
raise
else:
print("sysfs structures for %s%d created" % (sd.namestr(), prt))
print(f'sysfs structures for {sd.namestr()}{prt} created')
def create_disk_sysfs(dsk, first_sg, n):
sd = MySD(dsk)
@@ -314,17 +320,16 @@ def create_disk_sysfs(dsk, first_sg, n):
exec(disk_template.format(**disk))
except OSError:
si = sys.exc_info()[1]
if (si.errno == errno.EEXIST):
print("sysfs structures for %s exist" % sd.namestr())
elif (si.errno == errno.ENOENT):
print("error for %s: %s - have you run sys-script py first?" %
(sd.namestr(), si.strerror))
if si.errno == errno.EEXIST:
print(f'sysfs structures for {sd.namestr()} exist')
elif si.errno == errno.ENOENT:
print(f'error for {sd.namestr()}: {si.strerror} - have you run sys-script py first?')
return -1
else:
print("error for %s: %s" % (sd.namestr(), si.strerror))
print(f'error for {sd.namestr()}: {si.strerror}')
raise
else:
print("sysfs structures for %s created" % sd.namestr())
print(f'sysfs structures for {sd.namestr()} created')
n += 1
if n >= last:
@@ -338,6 +343,7 @@ def create_disk_sysfs(dsk, first_sg, n):
return n
os.chdir(sys.argv[1])
n = 0
last = int(sys.argv[2])
test/sys-script.py
+12 -5
View File
@@ -6,30 +6,36 @@
# © 2017 Canonical Ltd.
# Author: Dan Streetman <dan.streetman@canonical.com>
import os, sys
import os
import shutil
import sys
def d(path, mode):
os.mkdir(path, mode)
def l(path, src):
def l(path, src): # noqa: E743
os.symlink(src, path)
def f(path, mode, contents):
with open(path, "wb") as f:
with open(path, 'wb') as f:
f.write(contents)
os.chmod(path, mode)
if len(sys.argv) < 2:
exit("Usage: {} <target dir>".format(sys.argv[0]))
exit(f'Usage: {sys.argv[0]} <target dir>')
if not os.path.isdir(sys.argv[1]):
exit("Target dir {} not found".format(sys.argv[1]))
exit(f'Target dir {sys.argv[1]} not found')
os.chdir(sys.argv[1])
if os.path.exists('sys'):
shutil.rmtree('sys')
# fmt: off
d('sys', 0o755)
d('sys/kernel', 0o775)
f('sys/kernel/kexec_crash_loaded', 0o664, b'0\n')
@@ -16855,3 +16861,4 @@ f('sys/devices/platform/i8042/serio1/input/input1/capabilities/rel', 0o644, b'3\
f('sys/devices/platform/i8042/serio1/input/input1/capabilities/sw', 0o644, b'0\n')
f('sys/devices/platform/i8042/serio1/input/input1/capabilities/ev', 0o644, b'7\n')
f('sys/devices/platform/i8042/serio1/input/input1/capabilities/led', 0o644, b'0\n')
# fmt: on
test/test-network/systemd-networkd-tests.py
+2488 -1170
View File
File diff suppressed because it is too large Load Diff
test/test-shutdown.py
+54 -27
View File
@@ -13,27 +13,34 @@ import pexpect
def run(args):
ret = 1
logger = logging.getLogger("test-shutdown")
logger = logging.getLogger('test-shutdown')
logfile = None
if args.logfile:
logger.debug("Logging pexpect IOs to %s", args.logfile)
logger.debug('Logging pexpect IOs to %s', args.logfile)
logfile = open(args.logfile, 'w')
elif args.verbose:
logfile = sys.stdout
logger.info("spawning test")
console = pexpect.spawn(args.command, args.arg, logfile=logfile, env={
"TERM": "dumb",
}, encoding='utf-8', timeout=60)
logger.info('spawning test')
console = pexpect.spawn(
args.command,
args.arg,
logfile=logfile,
env={
'TERM': 'dumb',
},
encoding='utf-8',
timeout=60,
)
logger.debug("child pid %d", console.pid)
logger.debug('child pid %d', console.pid)
try:
logger.info("waiting for login prompt")
logger.info('waiting for login prompt')
console.expect('H login: ', 10)
logger.info("log in and start screen")
logger.info('log in and start screen')
console.sendline('root')
console.expect('bash.*# ', 10)
console.sendline('screen')
@@ -46,42 +53,42 @@ def run(args):
console.sendline('systemctl is-system-running --wait')
console.expect(r'\b(running|degraded)\b', 60)
# console.interact()
# console.interact()
console.sendline('tty')
console.expect(r'/dev/(pts/\d+)')
pty = console.match.group(1)
logger.info("window 1 at tty %s", pty)
logger.info('window 1 at tty %s', pty)
logger.info("schedule reboot")
logger.info('schedule reboot')
console.sendline('shutdown -r')
console.expect("Reboot scheduled for (?P<date>.*), use 'shutdown -c' to cancel", 2)
date = console.match.group('date')
logger.info("reboot scheduled for %s", date)
logger.info('reboot scheduled for %s', date)
console.sendcontrol('a')
console.send('0')
logger.info("verify broadcast message")
logger.info('verify broadcast message')
console.expect(f'Broadcast message from root@H on {pty}', 2)
console.expect(f'The system will reboot at {date}', 2)
logger.info("check show output")
logger.info('check show output')
console.sendline('shutdown --show')
console.expect(f"Reboot scheduled for {date}, use 'shutdown -c' to cancel", 2)
logger.info("cancel shutdown")
logger.info('cancel shutdown')
console.sendline('shutdown -c')
console.sendcontrol('a')
console.send('1')
console.expect('System shutdown has been cancelled', 2)
logger.info("call for reboot")
logger.info('call for reboot')
console.sendline('sleep 10; shutdown -r now')
console.sendcontrol('a')
console.send('0')
console.expect("The system will reboot now!", 12)
console.expect('The system will reboot now!', 12)
logger.info("waiting for reboot")
logger.info('waiting for reboot')
console.expect('H login: ', 60)
console.sendline('root')
@@ -89,16 +96,16 @@ def run(args):
console.sendline('> /testok')
logger.info("power off")
logger.info('power off')
console.sendline('poweroff')
logger.info("expect termination now")
logger.info('expect termination now')
console.expect(pexpect.EOF)
ret = 0
except Exception as e:
logger.error(e)
logger.info("killing child pid %d", console.pid)
logger.info('killing child pid %d', console.pid)
# Ask systemd-nspawn to stop and release the container's resources properly.
console.kill(signal.SIGTERM)
@@ -116,12 +123,31 @@ def run(args):
return ret
def main():
parser = argparse.ArgumentParser(description='test logind shutdown feature')
parser.add_argument("-v", "--verbose", action="store_true", help="verbose")
parser.add_argument("--logfile", metavar='FILE', help="Save all test input/output to the given path")
parser.add_argument("command", help="command to run")
parser.add_argument("arg", nargs='*', help="args for command")
parser = argparse.ArgumentParser(
description='test logind shutdown feature',
)
parser.add_argument(
'-v',
'--verbose',
action='store_true',
help='verbose',
)
parser.add_argument(
'--logfile',
metavar='FILE',
help='Save all test input/output to the given path',
)
parser.add_argument(
'command',
help='command to run',
)
parser.add_argument(
'arg',
nargs='*',
help='args for command',
)
args = parser.parse_args()
@@ -134,6 +160,7 @@ def main():
return run(args)
if __name__ == '__main__':
sys.exit(main())
test/test-systemd-tmpfiles.py
+74 -45
View File
@@ -6,13 +6,13 @@
# the Free Software Foundation; either version 2.1 of the License, or
# (at your option) any later version.
import grp
import os
import sys
import pwd
import socket
import subprocess
import sys
import tempfile
import pwd
import grp
from pathlib import Path
try:
@@ -20,40 +20,41 @@ try:
except ImportError:
id128 = None
EX_DATAERR = 65 # from sysexits.h
EX_DATAERR = 65 # from sysexits.h
EXIT_TEST_SKIP = 77
try:
subprocess.run
except AttributeError:
sys.exit(EXIT_TEST_SKIP)
exe_with_args = sys.argv[1:]
temp_dir = tempfile.TemporaryDirectory(prefix='test-systemd-tmpfiles.')
# If /tmp isn't owned by either 'root' or the current user
# systemd-tmpfiles will exit with "Detected unsafe path transition"
# breaking this test
tmpowner = os.stat("/tmp").st_uid
tmpowner = os.stat('/tmp').st_uid
if tmpowner != 0 and tmpowner != os.getuid():
print("Skip: /tmp is not owned by 'root' or current user")
sys.exit(EXIT_TEST_SKIP)
def test_line(line, *, user, returncode=EX_DATAERR, extra={}):
args = ['--user'] if user else []
print('Running {} on {!r}'.format(' '.join(exe_with_args + args), line))
c = subprocess.run(exe_with_args + ['--create', '-'] + args,
input=line, stdout=subprocess.PIPE, universal_newlines=True,
**extra)
c = subprocess.run(
exe_with_args + ['--create', '-'] + args,
input=line,
stdout=subprocess.PIPE,
text=True,
**extra,
)
assert c.returncode == returncode, c
def test_invalids(*, user):
test_line('asdfa', user=user)
test_line('f "open quote', user=user)
test_line('f closed quote""', user=user)
test_line('Y /unknown/letter', user=user)
test_line('w non/absolute/path', user=user)
test_line('s', user=user) # s is for short
test_line('s', user=user) # s is for short
test_line('f!! /too/many/bangs', user=user)
test_line('f++ /too/many/plusses', user=user)
test_line('f+!+ /too/many/plusses', user=user)
@@ -77,12 +78,18 @@ def test_invalids(*, user):
test_line('h - - -', user=user)
test_line('H - - -', user=user)
def test_uninitialized_t():
if os.getuid() == 0:
return
test_line('w /foo - - - - "specifier for --user %t"',
user=True, returncode=0, extra={'env':{'HOME': os.getenv('HOME')}})
test_line(
'w /foo - - - - "specifier for --user %t"',
user=True,
returncode=0,
extra={'env': {'HOME': os.getenv('HOME')}},
)
def test_content(line, expected, *, user, extra={}, subpath='/arg', path_cb=None):
d = tempfile.TemporaryDirectory(prefix='test-content.', dir=temp_dir.name)
@@ -92,24 +99,25 @@ def test_content(line, expected, *, user, extra={}, subpath='/arg', path_cb=None
spec = line.format(arg)
test_line(spec, user=user, returncode=0, extra=extra)
content = open(arg).read()
print('expect: {!r}\nactual: {!r}'.format(expected, content))
print(f'expect: {expected!r}\nactual: {content!r}')
assert content == expected
def test_valid_specifiers(*, user):
test_content('f {} - - - - two words', 'two words', user=user)
if id128 and os.path.isfile('/etc/machine-id'):
try:
test_content('f {} - - - - %m', '{}'.format(id128.get_machine().hex), user=user)
test_content('f {} - - - - %m', f'{id128.get_machine().hex}', user=user)
except AssertionError as e:
print(e)
print('/etc/machine-id: {!r}'.format(open('/etc/machine-id').read()))
print('/proc/cmdline: {!r}'.format(open('/proc/cmdline').read()))
print('skipping')
test_content('f {} - - - - %b', '{}'.format(id128.get_boot().hex), user=user)
test_content('f {} - - - - %H', '{}'.format(socket.gethostname()), user=user)
test_content('f {} - - - - %v', '{}'.format(os.uname().release), user=user)
test_content('f {} - - - - %U', '{}'.format(os.getuid() if user else 0), user=user)
test_content('f {} - - - - %G', '{}'.format(os.getgid() if user else 0), user=user)
test_content('f {} - - - - %b', f'{id128.get_boot().hex}', user=user)
test_content('f {} - - - - %H', f'{socket.gethostname()}', user=user)
test_content('f {} - - - - %v', f'{os.uname().release}', user=user)
test_content('f {} - - - - %U', f'{os.getuid() if user else 0}', user=user)
test_content('f {} - - - - %G', f'{os.getgid() if user else 0}', user=user)
try:
puser = pwd.getpwuid(os.getuid() if user else 0)
@@ -117,7 +125,7 @@ def test_valid_specifiers(*, user):
puser = None
if puser:
test_content('f {} - - - - %u', '{}'.format(puser.pw_name), user=user)
test_content('f {} - - - - %u', f'{puser.pw_name}', user=user)
try:
pgroup = grp.getgrgid(os.getgid() if user else 0)
@@ -125,52 +133,55 @@ def test_valid_specifiers(*, user):
pgroup = None
if pgroup:
test_content('f {} - - - - %g', '{}'.format(pgroup.gr_name), user=user)
test_content('f {} - - - - %g', f'{pgroup.gr_name}', user=user)
# Note that %h is the only specifier in which we look the environment,
# because we check $HOME. Should we even be doing that?
home = os.path.expanduser("~")
test_content('f {} - - - - %h', '{}'.format(home), user=user)
home = os.path.expanduser('~')
test_content('f {} - - - - %h', f'{home}', user=user)
xdg_runtime_dir = os.getenv('XDG_RUNTIME_DIR')
if xdg_runtime_dir is not None or not user:
test_content('f {} - - - - %t',
xdg_runtime_dir if user else '/run',
user=user)
user=user) # fmt: skip
xdg_state_home = os.getenv('XDG_STATE_HOME')
if xdg_state_home is None and user:
xdg_state_home = os.path.join(home, ".local/state")
xdg_state_home = os.path.join(home, '.local/state')
test_content('f {} - - - - %S',
xdg_state_home if user else '/var/lib',
user=user)
user=user) # fmt: skip
xdg_cache_home = os.getenv('XDG_CACHE_HOME')
if xdg_cache_home is None and user:
xdg_cache_home = os.path.join(home, ".cache")
xdg_cache_home = os.path.join(home, '.cache')
test_content('f {} - - - - %C',
xdg_cache_home if user else '/var/cache',
user=user)
user=user) # fmt: skip
test_content('f {} - - - - %L',
os.path.join(xdg_state_home, 'log') if user else '/var/log',
user=user)
user=user) # fmt: skip
test_content('f {} - - - - %%', '%', user=user)
def mkfifo(parent, subpath):
os.makedirs(parent, mode=0o755, exist_ok=True)
first_component = subpath.split('/')[1]
path = parent + '/' + first_component
print('path: {}'.format(path))
print(f'path: {path}')
os.mkfifo(path)
def mkdir(parent, subpath):
first_component = subpath.split('/')[1]
path = parent + '/' + first_component
os.makedirs(path, mode=0o755, exist_ok=True)
os.symlink(path, path + '/self', target_is_directory=True)
def symlink(parent, subpath):
link_path = parent + '/link-target'
os.makedirs(parent, mode=0o755, exist_ok=True)
@@ -180,6 +191,7 @@ def symlink(parent, subpath):
path = parent + '/' + first_component
os.symlink(link_path, path, target_is_directory=True)
def file(parent, subpath):
content = 'file-' + subpath.split('/')[1]
path = parent + subpath
@@ -187,6 +199,7 @@ def file(parent, subpath):
with open(path, 'wb') as f:
f.write(content.encode())
def valid_symlink(parent, subpath):
target = 'link-target'
link_path = parent + target
@@ -195,6 +208,7 @@ def valid_symlink(parent, subpath):
path = parent + '/' + first_component
os.symlink(target, path, target_is_directory=True)
def test_hard_cleanup(*, user):
type_cbs = [None, file, mkdir, symlink]
if 'mkfifo' in dir(os):
@@ -209,29 +223,44 @@ def test_hard_cleanup(*, user):
label = 'valid_symlink-deep'
test_content('f= {} - - - - ' + label, label, user=user, subpath='/deep/1/2', path_cb=valid_symlink)
def test_base64():
test_content('f~ {} - - - - UGlmZgpQYWZmClB1ZmYgCg==', "Piff\nPaff\nPuff \n", user=False)
test_content('f~ {} - - - - UGlmZgpQYWZmClB1ZmYgCg==', 'Piff\nPaff\nPuff \n', user=False)
def test_conditionalized_execute_bit():
c = subprocess.run(exe_with_args + ['--version', '|', 'grep', '-F', '+ACL'], shell=True, stdout=subprocess.DEVNULL)
c = subprocess.run(
exe_with_args + ['--version', '|', 'grep', '-F', '+ACL'],
shell=True,
stdout=subprocess.DEVNULL,
)
if c.returncode != 0:
return 0
d = tempfile.TemporaryDirectory(prefix='test-acl.', dir=temp_dir.name)
temp = Path(d.name) / "cond_exec"
temp = Path(d.name) / 'cond_exec'
temp.touch()
temp.chmod(0o644)
test_line(f"a {temp} - - - - u:root:Xwr", user=False, returncode=0)
c = subprocess.run(["getfacl", "-Ec", temp],
stdout=subprocess.PIPE, check=True, text=True)
assert "user:root:rw-" in c.stdout
test_line(f'a {temp} - - - - u:root:Xwr', user=False, returncode=0)
c = subprocess.run(
['getfacl', '-Ec', temp],
stdout=subprocess.PIPE,
check=True,
text=True,
)
assert 'user:root:rw-' in c.stdout
temp.chmod(0o755)
test_line(f"a+ {temp} - - - - u:root:Xwr,g:root:rX", user=False, returncode=0)
c = subprocess.run(["getfacl", "-Ec", temp],
stdout=subprocess.PIPE, check=True, text=True)
assert "user:root:rwx" in c.stdout and "group:root:r-x" in c.stdout
test_line(f'a+ {temp} - - - - u:root:Xwr,g:root:rX', user=False, returncode=0)
c = subprocess.run(
['getfacl', '-Ec', temp],
stdout=subprocess.PIPE,
check=True,
text=True,
)
assert 'user:root:rwx' in c.stdout and 'group:root:r-x' in c.stdout
if __name__ == '__main__':
test_invalids(user=False)
test/test-udev.py
+1191 -1181
View File
File diff suppressed because it is too large Load Diff
test/units/TEST-69-SHUTDOWN.py
+28 -26
View File
@@ -10,56 +10,58 @@ import pexpect
def main():
# TODO: drop once https://bugs.debian.org/1075733 is fixed
with open("/usr/lib/os-release") as f:
with open('/usr/lib/os-release') as f:
for line in f:
if line.startswith("ID="):
if "debian" in line or "ubuntu" in line:
if line.startswith('ID='):
if 'debian' in line or 'ubuntu' in line:
sys.exit(77)
logger = logging.getLogger("test-shutdown")
logger = logging.getLogger('test-shutdown')
consoles = []
for _ in range(2):
# Use script to allocate a separate pseudo tty to run the login shell in.
console = pexpect.spawn(
"script", ["--quiet", "--return", "--flush", "--command", "login -f root", "/dev/null"],
'script',
['--quiet', '--return', '--flush', '--command', 'login -f root', '/dev/null'],
logfile=sys.stdout,
env={"TERM": "dumb"},
encoding="utf-8",
env={'TERM': 'dumb'},
encoding='utf-8',
timeout=60,
)
logger.info("waiting for login prompt")
console.expect(".*# ", 10)
logger.info('waiting for login prompt')
console.expect('.*# ', 10)
consoles += [console]
consoles[1].sendline("tty")
consoles[1].expect(r"/dev/(pts/\d+)")
consoles[1].sendline('tty')
consoles[1].expect(r'/dev/(pts/\d+)')
pty = console.match.group(1)
logger.info("window 1 at tty %s", pty)
logger.info('window 1 at tty %s', pty)
logger.info("schedule reboot")
consoles[1].sendline("shutdown -r")
logger.info('schedule reboot')
consoles[1].sendline('shutdown -r')
consoles[1].expect("Reboot scheduled for (?P<date>.*), use 'shutdown -c' to cancel", 2)
date = consoles[1].match.group("date")
logger.info("reboot scheduled for %s", date)
date = consoles[1].match.group('date')
logger.info('reboot scheduled for %s', date)
logger.info("verify broadcast message")
consoles[0].expect(f"Broadcast message from root@H on {pty}", 2)
consoles[0].expect(f"The system will reboot at {date}!", 2)
logger.info('verify broadcast message')
consoles[0].expect(f'Broadcast message from root@H on {pty}', 2)
consoles[0].expect(f'The system will reboot at {date}!', 2)
logger.info("check show output")
consoles[1].sendline("shutdown --show")
logger.info('check show output')
consoles[1].sendline('shutdown --show')
consoles[1].expect(f"Reboot scheduled for {date}, use 'shutdown -c' to cancel", 2)
logger.info("cancel shutdown")
consoles[1].sendline("shutdown -c")
consoles[0].expect("System shutdown has been cancelled", 2)
logger.info('cancel shutdown')
consoles[1].sendline('shutdown -c')
consoles[0].expect('System shutdown has been cancelled', 2)
consoles[0].sendline("> /testok")
consoles[0].sendline('> /testok')
if __name__ == "__main__":
if __name__ == '__main__':
main()
# vim: sw=4 et
tools/analyze-dump-sort.py
+3 -2
View File
@@ -58,6 +58,7 @@ def sort_dump(sourcefile, destfile=None):
destfile.flush()
return destfile
def parse_args():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('one')
@@ -65,6 +66,7 @@ def parse_args():
p.add_argument('--user', action='store_true')
return p.parse_args()
if __name__ == '__main__':
opts = parse_args()
@@ -73,8 +75,7 @@ if __name__ == '__main__':
two = sort_dump(open(opts.two))
else:
user = ['--user'] if opts.user else []
two = subprocess.run(['systemd-analyze', 'dump', *user],
capture_output=True, text=True, check=True)
two = subprocess.run(['systemd-analyze', 'dump', *user], capture_output=True, text=True, check=True)
two = sort_dump(two.stdout.splitlines())
with subprocess.Popen(['diff', '-U10', one.name, two.name], stdout=subprocess.PIPE) as diff:
subprocess.Popen(['less'], stdin=diff.stdout)
tools/catalog-report.py
+9 -4
View File
@@ -36,9 +36,13 @@ from systemd import id128, journal
def log_entry(entry):
if 'CODE_FILE' in entry:
# some of our code was using 'CODE_FUNCTION' instead of 'CODE_FUNC'
print('{}:{} {}'.format(entry.get('CODE_FILE', '???'),
entry.get('CODE_LINE', '???'),
entry.get('CODE_FUNC', None) or entry.get('CODE_FUNCTION', '???')))
print(
'{}:{} {}'.format(
entry.get('CODE_FILE'),
entry.get('CODE_LINE', '???'),
entry.get('CODE_FUNC', None) or entry.get('CODE_FUNCTION', '???'),
)
)
print(' {}'.format(entry.get('MESSAGE', 'no message!')))
for k, v in entry.items():
if k.startswith('CODE_') or k in {'MESSAGE_ID', 'MESSAGE'}:
@@ -46,12 +50,13 @@ def log_entry(entry):
print(f' {k}={v}')
print()
if __name__ == '__main__':
j = journal.Reader()
logged = set()
pattern = re.compile('@[A-Z0-9_]+@')
mids = { v:k for k,v in id128.__dict__.items() if k.startswith('SD_MESSAGE') }
mids = {v: k for k, v in id128.__dict__.items() if k.startswith('SD_MESSAGE')}
for i, x in enumerate(j):
if i % 1000 == 0:
tools/check-efi-alignment.py
+6 -5
View File
@@ -15,22 +15,23 @@ def main():
pe = pefile.PE(sys.argv[1], fast_load=True)
for section in pe.sections:
name = section.Name.rstrip(b"\x00").decode()
name = section.Name.rstrip(b'\x00').decode()
file_addr = section.PointerToRawData
virt_addr = section.VirtualAddress
print(f"{name:10s} file=0x{file_addr:08x} virt=0x{virt_addr:08x}")
print(f'{name:10s} file=0x{file_addr:08x} virt=0x{virt_addr:08x}')
if file_addr % 512 != 0:
print(f"File address of {name} section is not aligned to 512 bytes", file=sys.stderr)
print(f'File address of {name} section is not aligned to 512 bytes', file=sys.stderr)
return 1
if virt_addr % 512 != 0:
print(f"Virt address of {name} section is not aligned to 512 bytes", file=sys.stderr)
print(f'Virt address of {name} section is not aligned to 512 bytes', file=sys.stderr)
return 1
if __name__ == '__main__':
if len(sys.argv) != 2:
print(f"Usage: {sys.argv[0]} pe-image")
print(f'Usage: {sys.argv[0]} pe-image')
sys.exit(1)
sys.exit(main())
tools/check-version-history.py
+34 -49
View File
@@ -20,17 +20,14 @@ def find_undocumented_functions(pages, ignorelist):
filename = os.path.basename(page)
pagetree = tree.parse(page)
assert pagetree.getroot().tag == "refentry"
assert pagetree.getroot().tag == 'refentry'
hist_section = pagetree.find("refsect1[title='History']")
for func in pagetree.findall(".//funcprototype/funcdef/function"):
for func in pagetree.findall('.//funcprototype/funcdef/function'):
path = f"./refsynopsisdiv/funcsynopsis/funcprototype/funcdef/function[.='{func.text}']"
assert pagetree.findall(path) == [func]
if (
hist_section is None
or hist_section.find(f"para/function[.='{func.text}()']") is None
):
if hist_section is None or hist_section.find(f"para/function[.='{func.text}()']") is None:
if func.text not in ignorelist:
undocumented.append((filename, func.text))
return undocumented
@@ -39,22 +36,22 @@ def find_undocumented_functions(pages, ignorelist):
def construct_path(element):
tag = element.tag
if tag == "refentry":
return "."
if tag == 'refentry':
return '.'
predicate = ""
if tag == "varlistentry":
text = "".join(element.find("term").itertext())
predicate = ''
if tag == 'varlistentry':
text = ''.join(element.find('term').itertext())
predicate = f'[term="{text}"]'
elif tag.startswith("refsect"):
text = "".join(element.find("title").itertext())
elif tag.startswith('refsect'):
text = ''.join(element.find('title').itertext())
predicate = f'[title="{text}"]'
elif tag == "variablelist":
elif tag == 'variablelist':
varlists = element.getparent().findall(tag)
if len(varlists) > 1:
predicate = f"[{varlists.index(element)+1}]"
predicate = f'[{varlists.index(element) + 1}]'
return construct_path(element.getparent()) + "/" + tag + predicate
return construct_path(element.getparent()) + '/' + tag + predicate
def find_undocumented_commands(pages, ignorelist):
@@ -63,23 +60,21 @@ def find_undocumented_commands(pages, ignorelist):
filename = os.path.basename(page)
pagetree = tree.parse(page)
if pagetree.getroot().tag != "refentry":
if pagetree.getroot().tag != 'refentry':
continue
for varlistentry in pagetree.findall("*//variablelist/varlistentry"):
for varlistentry in pagetree.findall('*//variablelist/varlistentry'):
path = construct_path(varlistentry)
assert pagetree.findall(path) == [varlistentry]
listitem = varlistentry.find("listitem")
listitem = varlistentry.find('listitem')
parent = listitem if listitem is not None else varlistentry
rev = parent.getchildren()[-1]
if (
rev.get("href") != "version-info.xml" and
not path.startswith(tuple(entry[1] for entry in ignorelist if entry[0] == filename))
):
undocumented.append((filename, path))
ignored_files = tuple(entry[1] for entry in ignorelist if entry[0] == filename)
if rev.get('href') != 'version-info.xml' and not path.startswith(ignored_files):
undocumented.append((filename, path))
return undocumented
@@ -90,54 +85,44 @@ def process_pages(pages):
for page in pages:
filename = os.path.basename(page)
if filename.startswith("org.freedesktop."): # dbus
if filename.startswith('org.freedesktop.'): # dbus
continue
if (
filename.startswith("sd_")
or filename.startswith("sd-")
or filename.startswith("udev_")
):
if filename.startswith('sd_') or filename.startswith('sd-') or filename.startswith('udev_'):
function_pages.append(page)
continue
command_pages.append(page)
undocumented_commands = find_undocumented_commands(
command_pages, command_ignorelist
)
undocumented_functions = find_undocumented_functions(
function_pages, function_ignorelist
)
undocumented_commands = find_undocumented_commands(command_pages, command_ignorelist)
undocumented_functions = find_undocumented_functions(function_pages, function_ignorelist)
return undocumented_commands, undocumented_functions
if __name__ == "__main__":
with open(os.path.join(os.path.dirname(__file__), "command_ignorelist")) as f:
if __name__ == '__main__':
with open(os.path.join(os.path.dirname(__file__), 'command_ignorelist')) as f:
command_ignorelist = []
for l in f.read().splitlines():
if l.startswith("#"):
for line in f.read().splitlines():
if line.startswith('#'):
continue
fname, path = l.split(" ", 1)
path = path.replace("\\n", "\n")
fname, path = line.split(' ', 1)
path = path.replace('\\n', '\n')
command_ignorelist.append((fname, path))
with open(os.path.join(os.path.dirname(__file__), "function_ignorelist")) as f:
with open(os.path.join(os.path.dirname(__file__), 'function_ignorelist')) as f:
function_ignorelist = f.read().splitlines()
undocumented_commands, undocumented_functions = process_pages(sys.argv[1:])
if undocumented_commands or undocumented_functions:
for filename, func in undocumented_functions:
print(
f"Function {func}() in {filename} isn't documented in the History section."
)
print(f"Function {func}() in {filename} isn't documented in the History section.")
for filename, path in undocumented_commands:
print(filename, path, "is undocumented")
print(filename, path, 'is undocumented')
if undocumented_commands:
print(
"Hint: if you reorganized this part of the documentation, "
"please update tools/commands_ignorelist."
'Hint: if you reorganized this part of the documentation, '
'please update tools/commands_ignorelist.'
)
sys.exit(1)
tools/dbus_exporter.py
+16 -8
View File
@@ -11,14 +11,15 @@ def extract_interfaces_xml(output_dir, executable):
# as glibc looks at /proc/self/exe when resolving RPATH
env = os.environ.copy()
if not os.path.exists('/proc/self'):
env["LD_ORIGIN_PATH"] = executable.parent.as_posix()
env['LD_ORIGIN_PATH'] = executable.parent.as_posix()
proc = run(
args=[executable.absolute(), '--bus-introspect', 'list'],
stdout=PIPE,
env=env,
check=True,
universal_newlines=True)
text=True,
)
interface_names = (x.split()[1] for x in proc.stdout.splitlines())
@@ -28,19 +29,25 @@ def extract_interfaces_xml(output_dir, executable):
stdout=PIPE,
env=env,
check=True,
universal_newlines=True)
text=True,
)
interface_file_name = output_dir / (interface_name + '.xml')
interface_file_name.write_text(proc.stdout)
interface_file_name.chmod(0o644)
def main():
parser = ArgumentParser()
parser.add_argument('output',
type=Path)
parser.add_argument('executables',
nargs='+',
type=Path)
parser.add_argument(
'output',
type=Path,
)
parser.add_argument(
'executables',
nargs='+',
type=Path,
)
args = parser.parse_args()
@@ -50,5 +57,6 @@ def main():
for exe in args.executables:
extract_interfaces_xml(args.output, exe)
if __name__ == '__main__':
main()
tools/dump-auxv.py
+37 -17
View File
@@ -85,24 +85,43 @@ AT_AUXV = {
'AT_L3_CACHEGEOMETRY' : 47,
'AT_MINSIGSTKSZ' : 51, # Stack needed for signal delivery
}
AT_AUXV_NAMES = {v:k for k,v in AT_AUXV.items()}
} # fmt: skip
AT_AUXV_NAMES = {v: k for k, v in AT_AUXV.items()}
@click.command(help=__doc__)
@click.option('-b', '--big-endian', 'endian',
flag_value='>',
help='Input is big-endian')
@click.option('-l', '--little-endian', 'endian',
flag_value='<',
help='Input is little-endian')
@click.option('-3', '--32', 'field_width',
flag_value=32,
help='Input is 32-bit')
@click.option('-6', '--64', 'field_width',
flag_value=64,
help='Input is 64-bit')
@click.argument('file',
type=click.File(mode='rb'))
@click.option(
'-b',
'--big-endian',
'endian',
flag_value='>',
help='Input is big-endian',
)
@click.option(
'-l',
'--little-endian',
'endian',
flag_value='<',
help='Input is little-endian',
)
@click.option(
'-3',
'--32',
'field_width',
flag_value=32,
help='Input is 32-bit',
)
@click.option(
'-6',
'--64',
'field_width',
flag_value=64,
help='Input is 64-bit',
)
@click.argument(
'file',
type=click.File(mode='rb'),
)
def dump(endian, field_width, file):
data = file.read()
@@ -111,7 +130,7 @@ def dump(endian, field_width, file):
if endian is None:
endian = '@'
width = {32:'II', 64:'QQ'}[field_width]
width = {32: 'II', 64: 'QQ'}[field_width]
format_str = f'{endian}{width}'
print(f'# {format_str=}')
@@ -137,5 +156,6 @@ def dump(endian, field_width, file):
if not seen_null:
print('# array not terminated with AT_NULL')
if __name__ == '__main__':
dump()
tools/elf2efi.py
+235 -236
View File
@@ -31,12 +31,12 @@ import sys
import time
import typing
from ctypes import (
LittleEndianStructure,
c_char,
c_uint8,
c_uint16,
c_uint32,
c_uint64,
LittleEndianStructure,
sizeof,
)
@@ -46,28 +46,28 @@ from elftools.elf import elffile
class PeCoffHeader(LittleEndianStructure):
_fields_ = (
("Machine", c_uint16),
("NumberOfSections", c_uint16),
("TimeDateStamp", c_uint32),
("PointerToSymbolTable", c_uint32),
("NumberOfSymbols", c_uint32),
("SizeOfOptionalHeader", c_uint16),
("Characteristics", c_uint16),
)
('Machine', c_uint16),
('NumberOfSections', c_uint16),
('TimeDateStamp', c_uint32),
('PointerToSymbolTable', c_uint32),
('NumberOfSymbols', c_uint32),
('SizeOfOptionalHeader', c_uint16),
('Characteristics', c_uint16),
) # fmt: skip
class PeDataDirectory(LittleEndianStructure):
_fields_ = (
("VirtualAddress", c_uint32),
("Size", c_uint32),
)
('VirtualAddress', c_uint32),
('Size', c_uint32),
) # fmt: skip
class PeRelocationBlock(LittleEndianStructure):
_fields_ = (
("PageRVA", c_uint32),
("BlockSize", c_uint32),
)
('PageRVA', c_uint32),
('BlockSize', c_uint32),
) # fmt: skip
def __init__(self, PageRVA: int):
super().__init__(PageRVA)
@@ -76,64 +76,64 @@ class PeRelocationBlock(LittleEndianStructure):
class PeRelocationEntry(LittleEndianStructure):
_fields_ = (
("Offset", c_uint16, 12),
("Type", c_uint16, 4),
)
('Offset', c_uint16, 12),
('Type', c_uint16, 4),
) # fmt: skip
class PeOptionalHeaderStart(LittleEndianStructure):
_fields_ = (
("Magic", c_uint16),
("MajorLinkerVersion", c_uint8),
("MinorLinkerVersion", c_uint8),
("SizeOfCode", c_uint32),
("SizeOfInitializedData", c_uint32),
("SizeOfUninitializedData", c_uint32),
("AddressOfEntryPoint", c_uint32),
("BaseOfCode", c_uint32),
)
('Magic', c_uint16),
('MajorLinkerVersion', c_uint8),
('MinorLinkerVersion', c_uint8),
('SizeOfCode', c_uint32),
('SizeOfInitializedData', c_uint32),
('SizeOfUninitializedData', c_uint32),
('AddressOfEntryPoint', c_uint32),
('BaseOfCode', c_uint32),
) # fmt: skip
class PeOptionalHeaderMiddle(LittleEndianStructure):
_fields_ = (
("SectionAlignment", c_uint32),
("FileAlignment", c_uint32),
("MajorOperatingSystemVersion", c_uint16),
("MinorOperatingSystemVersion", c_uint16),
("MajorImageVersion", c_uint16),
("MinorImageVersion", c_uint16),
("MajorSubsystemVersion", c_uint16),
("MinorSubsystemVersion", c_uint16),
("Win32VersionValue", c_uint32),
("SizeOfImage", c_uint32),
("SizeOfHeaders", c_uint32),
("CheckSum", c_uint32),
("Subsystem", c_uint16),
("DllCharacteristics", c_uint16),
)
('SectionAlignment', c_uint32),
('FileAlignment', c_uint32),
('MajorOperatingSystemVersion', c_uint16),
('MinorOperatingSystemVersion', c_uint16),
('MajorImageVersion', c_uint16),
('MinorImageVersion', c_uint16),
('MajorSubsystemVersion', c_uint16),
('MinorSubsystemVersion', c_uint16),
('Win32VersionValue', c_uint32),
('SizeOfImage', c_uint32),
('SizeOfHeaders', c_uint32),
('CheckSum', c_uint32),
('Subsystem', c_uint16),
('DllCharacteristics', c_uint16),
) # fmt: skip
class PeOptionalHeaderEnd(LittleEndianStructure):
_fields_ = (
("LoaderFlags", c_uint32),
("NumberOfRvaAndSizes", c_uint32),
("ExportTable", PeDataDirectory),
("ImportTable", PeDataDirectory),
("ResourceTable", PeDataDirectory),
("ExceptionTable", PeDataDirectory),
("CertificateTable", PeDataDirectory),
("BaseRelocationTable", PeDataDirectory),
("Debug", PeDataDirectory),
("Architecture", PeDataDirectory),
("GlobalPtr", PeDataDirectory),
("TLSTable", PeDataDirectory),
("LoadConfigTable", PeDataDirectory),
("BoundImport", PeDataDirectory),
("IAT", PeDataDirectory),
("DelayImportDescriptor", PeDataDirectory),
("CLRRuntimeHeader", PeDataDirectory),
("Reserved", PeDataDirectory),
)
('LoaderFlags', c_uint32),
('NumberOfRvaAndSizes', c_uint32),
('ExportTable', PeDataDirectory),
('ImportTable', PeDataDirectory),
('ResourceTable', PeDataDirectory),
('ExceptionTable', PeDataDirectory),
('CertificateTable', PeDataDirectory),
('BaseRelocationTable', PeDataDirectory),
('Debug', PeDataDirectory),
('Architecture', PeDataDirectory),
('GlobalPtr', PeDataDirectory),
('TLSTable', PeDataDirectory),
('LoadConfigTable', PeDataDirectory),
('BoundImport', PeDataDirectory),
('IAT', PeDataDirectory),
('DelayImportDescriptor', PeDataDirectory),
('CLRRuntimeHeader', PeDataDirectory),
('Reserved', PeDataDirectory),
) # fmt: skip
class PeOptionalHeader(LittleEndianStructure):
@@ -141,47 +141,47 @@ class PeOptionalHeader(LittleEndianStructure):
class PeOptionalHeader32(PeOptionalHeader):
_anonymous_ = ("Start", "Middle", "End")
_anonymous_ = ('Start', 'Middle', 'End')
_fields_ = (
("Start", PeOptionalHeaderStart),
("BaseOfData", c_uint32),
("ImageBase", c_uint32),
("Middle", PeOptionalHeaderMiddle),
("SizeOfStackReserve", c_uint32),
("SizeOfStackCommit", c_uint32),
("SizeOfHeapReserve", c_uint32),
("SizeOfHeapCommit", c_uint32),
("End", PeOptionalHeaderEnd),
)
('Start', PeOptionalHeaderStart),
('BaseOfData', c_uint32),
('ImageBase', c_uint32),
('Middle', PeOptionalHeaderMiddle),
('SizeOfStackReserve', c_uint32),
('SizeOfStackCommit', c_uint32),
('SizeOfHeapReserve', c_uint32),
('SizeOfHeapCommit', c_uint32),
('End', PeOptionalHeaderEnd),
) # fmt: skip
class PeOptionalHeader32Plus(PeOptionalHeader):
_anonymous_ = ("Start", "Middle", "End")
_anonymous_ = ('Start', 'Middle', 'End')
_fields_ = (
("Start", PeOptionalHeaderStart),
("ImageBase", c_uint64),
("Middle", PeOptionalHeaderMiddle),
("SizeOfStackReserve", c_uint64),
("SizeOfStackCommit", c_uint64),
("SizeOfHeapReserve", c_uint64),
("SizeOfHeapCommit", c_uint64),
("End", PeOptionalHeaderEnd),
)
('Start', PeOptionalHeaderStart),
('ImageBase', c_uint64),
('Middle', PeOptionalHeaderMiddle),
('SizeOfStackReserve', c_uint64),
('SizeOfStackCommit', c_uint64),
('SizeOfHeapReserve', c_uint64),
('SizeOfHeapCommit', c_uint64),
('End', PeOptionalHeaderEnd),
) # fmt: skip
class PeSection(LittleEndianStructure):
_fields_ = (
("Name", c_char * 8),
("VirtualSize", c_uint32),
("VirtualAddress", c_uint32),
("SizeOfRawData", c_uint32),
("PointerToRawData", c_uint32),
("PointerToRelocations", c_uint32),
("PointerToLinenumbers", c_uint32),
("NumberOfRelocations", c_uint16),
("NumberOfLinenumbers", c_uint16),
("Characteristics", c_uint32),
)
('Name', c_char * 8),
('VirtualSize', c_uint32),
('VirtualAddress', c_uint32),
('SizeOfRawData', c_uint32),
('PointerToRawData', c_uint32),
('PointerToRelocations', c_uint32),
('PointerToLinenumbers', c_uint32),
('NumberOfRelocations', c_uint16),
('NumberOfLinenumbers', c_uint16),
('Characteristics', c_uint32),
) # fmt: skip
def __init__(self) -> None:
super().__init__()
@@ -195,30 +195,32 @@ assert sizeof(PeCoffHeader) == 20
assert sizeof(PeOptionalHeader32) == 224
assert sizeof(PeOptionalHeader32Plus) == 240
# fmt: off
PE_CHARACTERISTICS_RX = 0x60000020 # CNT_CODE|MEM_READ|MEM_EXECUTE
PE_CHARACTERISTICS_RW = 0xC0000040 # CNT_INITIALIZED_DATA|MEM_READ|MEM_WRITE
PE_CHARACTERISTICS_R = 0x40000040 # CNT_INITIALIZED_DATA|MEM_READ
# fmt: on
IGNORE_SECTIONS = [
".eh_frame",
".eh_frame_hdr",
".ARM.exidx",
".relro_padding",
".sframe",
'.eh_frame',
'.eh_frame_hdr',
'.ARM.exidx',
'.relro_padding',
'.sframe',
]
IGNORE_SECTION_TYPES = [
"SHT_DYNAMIC",
"SHT_DYNSYM",
"SHT_GNU_ATTRIBUTES",
"SHT_GNU_HASH",
"SHT_HASH",
"SHT_NOTE",
"SHT_REL",
"SHT_RELA",
"SHT_RELR",
"SHT_STRTAB",
"SHT_SYMTAB",
'SHT_DYNAMIC',
'SHT_DYNSYM',
'SHT_GNU_ATTRIBUTES',
'SHT_GNU_HASH',
'SHT_HASH',
'SHT_NOTE',
'SHT_REL',
'SHT_RELA',
'SHT_RELR',
'SHT_STRTAB',
'SHT_SYMTAB',
]
# EFI mandates 4KiB memory pages.
@@ -227,7 +229,7 @@ FILE_ALIGNMENT = 512
# Nobody cares about DOS headers, so put the PE header right after.
PE_OFFSET = 64
PE_MAGIC = b"PE\0\0"
PE_MAGIC = b'PE\0\0'
def align_to(x: int, align: int) -> int:
@@ -239,8 +241,7 @@ def align_down(x: int, align: int) -> int:
def next_section_address(sections: list[PeSection]) -> int:
return align_to(sections[-1].VirtualAddress + sections[-1].VirtualSize,
SECTION_ALIGNMENT)
return align_to(sections[-1].VirtualAddress + sections[-1].VirtualSize, SECTION_ALIGNMENT)
class BadSectionError(ValueError):
@@ -256,29 +257,30 @@ def iter_copy_sections(file: elffile.ELFFile) -> typing.Iterator[PeSection]:
relro = None
for elf_seg in file.iter_segments():
if elf_seg["p_type"] == "PT_LOAD" and elf_seg["p_align"] != SECTION_ALIGNMENT:
raise BadSectionError(f"ELF segment {elf_seg['p_type']} is not properly aligned"
f" ({elf_seg['p_align']} != {SECTION_ALIGNMENT})")
if elf_seg["p_type"] == "PT_GNU_RELRO":
if elf_seg['p_type'] == 'PT_LOAD' and elf_seg['p_align'] != SECTION_ALIGNMENT:
raise BadSectionError(
f'ELF segment {elf_seg["p_type"]} is not properly aligned ({elf_seg["p_align"]} != {SECTION_ALIGNMENT})'
)
if elf_seg['p_type'] == 'PT_GNU_RELRO':
relro = elf_seg
for elf_s in file.iter_sections():
if (
elf_s["sh_flags"] & elf.constants.SH_FLAGS.SHF_ALLOC == 0
or elf_s["sh_type"] in IGNORE_SECTION_TYPES
elf_s['sh_flags'] & elf.constants.SH_FLAGS.SHF_ALLOC == 0
or elf_s['sh_type'] in IGNORE_SECTION_TYPES
or elf_s.name in IGNORE_SECTIONS
or elf_s["sh_size"] == 0
or elf_s['sh_size'] == 0
):
continue
if elf_s["sh_type"] not in ["SHT_PROGBITS", "SHT_NOBITS"]:
raise BadSectionError(f"Unknown section {elf_s.name} with type {elf_s['sh_type']}")
if elf_s['sh_type'] not in ['SHT_PROGBITS', 'SHT_NOBITS']:
raise BadSectionError(f'Unknown section {elf_s.name} with type {elf_s["sh_type"]}')
if elf_s.name == '.got':
# FIXME: figure out why those sections are inserted
print("WARNING: Non-empty .got section", file=sys.stderr)
print('WARNING: Non-empty .got section', file=sys.stderr)
if elf_s["sh_flags"] & elf.constants.SH_FLAGS.SHF_EXECINSTR:
if elf_s['sh_flags'] & elf.constants.SH_FLAGS.SHF_EXECINSTR:
rwx = PE_CHARACTERISTICS_RX
elif elf_s["sh_flags"] & elf.constants.SH_FLAGS.SHF_WRITE:
elif elf_s['sh_flags'] & elf.constants.SH_FLAGS.SHF_WRITE:
rwx = PE_CHARACTERISTICS_RW
else:
rwx = PE_CHARACTERISTICS_R
@@ -293,11 +295,11 @@ def iter_copy_sections(file: elffile.ELFFile) -> typing.Iterator[PeSection]:
if pe_s:
# Insert padding to properly align the section.
pad_len = elf_s["sh_addr"] - pe_s.VirtualAddress - len(pe_s.data)
pad_len = elf_s['sh_addr'] - pe_s.VirtualAddress - len(pe_s.data)
pe_s.data += bytearray(pad_len) + elf_s.data()
else:
pe_s = PeSection()
pe_s.VirtualAddress = elf_s["sh_addr"]
pe_s.VirtualAddress = elf_s['sh_addr']
pe_s.Characteristics = rwx
pe_s.data = elf_s.data()
@@ -306,8 +308,8 @@ def iter_copy_sections(file: elffile.ELFFile) -> typing.Iterator[PeSection]:
def convert_sections(
file: elffile.ELFFile,
opt: PeOptionalHeader,
file: elffile.ELFFile,
opt: PeOptionalHeader,
) -> list[PeSection]:
last_vma = (0, 0)
sections = []
@@ -324,24 +326,25 @@ def convert_sections(
pe_s.VirtualSize = len(pe_s.data)
pe_s.SizeOfRawData = align_to(len(pe_s.data), FILE_ALIGNMENT)
pe_s.Name = {
PE_CHARACTERISTICS_RX: b".text",
PE_CHARACTERISTICS_RW: b".data",
PE_CHARACTERISTICS_R: b".rodata",
PE_CHARACTERISTICS_RX: b'.text',
PE_CHARACTERISTICS_RW: b'.data',
PE_CHARACTERISTICS_R: b'.rodata',
}[pe_s.Characteristics]
# This can happen if not building with '-z separate-code'.
if pe_s.VirtualAddress < sum(last_vma):
raise BadSectionError(f"Section {pe_s.Name.decode()!r} @0x{pe_s.VirtualAddress:x} overlaps"
f" previous section @0x{last_vma[0]:x}+0x{last_vma[1]:x}=@0x{sum(last_vma):x}")
raise BadSectionError(
f'Section {pe_s.Name.decode()!r} @{pe_s.VirtualAddress:#x} overlaps previous section @{last_vma[0]:#x}+{last_vma[1]:#x}=@{sum(last_vma):#x}'
)
last_vma = (pe_s.VirtualAddress, pe_s.VirtualSize)
if pe_s.Name == b".text":
if pe_s.Name == b'.text':
opt.BaseOfCode = pe_s.VirtualAddress
opt.SizeOfCode += pe_s.VirtualSize
else:
opt.SizeOfInitializedData += pe_s.VirtualSize
if pe_s.Name == b".data" and isinstance(opt, PeOptionalHeader32):
if pe_s.Name == b'.data' and isinstance(opt, PeOptionalHeader32):
opt.BaseOfData = pe_s.VirtualAddress
sections.append(pe_s)
@@ -355,14 +358,14 @@ def copy_sections(
input_names: str,
sections: list[PeSection],
) -> None:
for name in input_names.split(","):
for name in input_names.split(','):
elf_s = file.get_section_by_name(name)
if not elf_s:
continue
if elf_s.data_alignment > 1 and SECTION_ALIGNMENT % elf_s.data_alignment != 0:
raise BadSectionError(f"ELF section {name} is not aligned")
if elf_s["sh_flags"] & (elf.constants.SH_FLAGS.SHF_EXECINSTR | elf.constants.SH_FLAGS.SHF_WRITE) != 0:
raise BadSectionError(f"ELF section {name} is not read-only data")
raise BadSectionError(f'ELF section {name} is not aligned')
if elf_s['sh_flags'] & (elf.constants.SH_FLAGS.SHF_EXECINSTR | elf.constants.SH_FLAGS.SHF_WRITE) != 0: # fmt: skip
raise BadSectionError(f'ELF section {name} is not read-only data')
pe_s = PeSection()
pe_s.Name = name.encode()
@@ -381,18 +384,21 @@ def apply_elf_relative_relocation(
sections: list[PeSection],
addend_size: int,
) -> None:
[target] = [pe_s for pe_s in sections
if pe_s.VirtualAddress <= reloc["r_offset"] < pe_s.VirtualAddress + len(pe_s.data)]
[target] = [
pe_s
for pe_s in sections
if pe_s.VirtualAddress <= reloc['r_offset'] < pe_s.VirtualAddress + len(pe_s.data)
]
addend_offset = reloc["r_offset"] - target.VirtualAddress
addend_offset = reloc['r_offset'] - target.VirtualAddress
if reloc.is_RELA():
addend = reloc["r_addend"]
addend = reloc['r_addend']
else:
addend = target.data[addend_offset : addend_offset + addend_size]
addend = int.from_bytes(addend, byteorder="little")
addend = int.from_bytes(addend, byteorder='little')
value = (image_base + addend).to_bytes(addend_size, byteorder="little")
value = (image_base + addend).to_bytes(addend_size, byteorder='little')
target.data[addend_offset : addend_offset + addend_size] = value
@@ -404,47 +410,44 @@ def convert_elf_reloc_table(
pe_reloc_blocks: dict[int, PeRelocationBlock],
) -> None:
NONE_RELOC = {
"EM_386": elf.enums.ENUM_RELOC_TYPE_i386["R_386_NONE"],
"EM_AARCH64": elf.enums.ENUM_RELOC_TYPE_AARCH64["R_AARCH64_NONE"],
"EM_ARM": elf.enums.ENUM_RELOC_TYPE_ARM["R_ARM_NONE"],
"EM_LOONGARCH": 0,
"EM_RISCV": 0,
"EM_X86_64": elf.enums.ENUM_RELOC_TYPE_x64["R_X86_64_NONE"],
}[file["e_machine"]]
'EM_386': elf.enums.ENUM_RELOC_TYPE_i386['R_386_NONE'],
'EM_AARCH64': elf.enums.ENUM_RELOC_TYPE_AARCH64['R_AARCH64_NONE'],
'EM_ARM': elf.enums.ENUM_RELOC_TYPE_ARM['R_ARM_NONE'],
'EM_LOONGARCH': 0,
'EM_RISCV': 0,
'EM_X86_64': elf.enums.ENUM_RELOC_TYPE_x64['R_X86_64_NONE'],
}[file['e_machine']] # fmt: skip
RELATIVE_RELOC = {
"EM_386": elf.enums.ENUM_RELOC_TYPE_i386["R_386_RELATIVE"],
"EM_AARCH64": elf.enums.ENUM_RELOC_TYPE_AARCH64["R_AARCH64_RELATIVE"],
"EM_ARM": elf.enums.ENUM_RELOC_TYPE_ARM["R_ARM_RELATIVE"],
"EM_LOONGARCH": 3,
"EM_RISCV": 3,
"EM_X86_64": elf.enums.ENUM_RELOC_TYPE_x64["R_X86_64_RELATIVE"],
}[file["e_machine"]]
'EM_386': elf.enums.ENUM_RELOC_TYPE_i386['R_386_RELATIVE'],
'EM_AARCH64': elf.enums.ENUM_RELOC_TYPE_AARCH64['R_AARCH64_RELATIVE'],
'EM_ARM': elf.enums.ENUM_RELOC_TYPE_ARM['R_ARM_RELATIVE'],
'EM_LOONGARCH': 3,
'EM_RISCV': 3,
'EM_X86_64': elf.enums.ENUM_RELOC_TYPE_x64['R_X86_64_RELATIVE'],
}[file['e_machine']] # fmt: skip
for reloc in elf_reloc_table.iter_relocations():
if reloc["r_info_type"] == NONE_RELOC:
if reloc['r_info_type'] == NONE_RELOC:
continue
if reloc["r_info_type"] == RELATIVE_RELOC:
apply_elf_relative_relocation(reloc,
elf_image_base,
sections,
file.elfclass // 8)
if reloc['r_info_type'] == RELATIVE_RELOC:
apply_elf_relative_relocation(reloc, elf_image_base, sections, file.elfclass // 8)
# Now that the ELF relocation has been applied, we can create a PE relocation.
block_rva = reloc["r_offset"] & ~0xFFF
block_rva = reloc['r_offset'] & ~0xFFF
if block_rva not in pe_reloc_blocks:
pe_reloc_blocks[block_rva] = PeRelocationBlock(block_rva)
entry = PeRelocationEntry()
entry.Offset = reloc["r_offset"] & 0xFFF
entry.Offset = reloc['r_offset'] & 0xFFF
# REL_BASED_HIGHLOW or REL_BASED_DIR64
entry.Type = 3 if file.elfclass == 32 else 10
pe_reloc_blocks[block_rva].entries.append(entry)
continue
raise BadSectionError(f"Unsupported relocation {reloc}")
raise BadSectionError(f'Unsupported relocation {reloc}')
def convert_elf_relocations(
@@ -453,27 +456,29 @@ def convert_elf_relocations(
sections: list[PeSection],
minimum_sections: int,
) -> typing.Optional[PeSection]:
dynamic = file.get_section_by_name(".dynamic")
dynamic = file.get_section_by_name('.dynamic')
if dynamic is None:
raise BadSectionError("ELF .dynamic section is missing")
raise BadSectionError('ELF .dynamic section is missing')
[flags_tag] = dynamic.iter_tags("DT_FLAGS_1")
if not flags_tag["d_val"] & elf.enums.ENUM_DT_FLAGS_1["DF_1_PIE"]:
raise ValueError("ELF file is not a PIE")
[flags_tag] = dynamic.iter_tags('DT_FLAGS_1')
if not flags_tag['d_val'] & elf.enums.ENUM_DT_FLAGS_1['DF_1_PIE']:
raise ValueError('ELF file is not a PIE')
# This checks that the ELF image base is 0.
symtab = file.get_section_by_name(".symtab")
symtab = file.get_section_by_name('.symtab')
if symtab:
exe_start = symtab.get_symbol_by_name("__executable_start")
if exe_start and exe_start[0]["st_value"] != 0:
raise ValueError("Unexpected ELF image base")
exe_start = symtab.get_symbol_by_name('__executable_start')
if exe_start and exe_start[0]['st_value'] != 0:
raise ValueError('Unexpected ELF image base')
opt.SizeOfHeaders = align_to(PE_OFFSET
+ len(PE_MAGIC)
+ sizeof(PeCoffHeader)
+ sizeof(opt)
+ sizeof(PeSection) * max(len(sections) + 1, minimum_sections),
FILE_ALIGNMENT)
opt.SizeOfHeaders = align_to(
PE_OFFSET
+ len(PE_MAGIC)
+ sizeof(PeCoffHeader)
+ sizeof(opt)
+ sizeof(PeSection) * max(len(sections) + 1, minimum_sections),
FILE_ALIGNMENT,
)
# We use the basic VMA layout from the ELF image in the PE image. This could cause the first
# section to overlap the PE image headers during runtime at VMA 0. We can simply apply a fixed
@@ -482,23 +487,18 @@ def convert_elf_relocations(
# the ELF portions of the image.
segment_offset = 0
if sections[0].VirtualAddress < opt.SizeOfHeaders:
segment_offset = align_to(opt.SizeOfHeaders - sections[0].VirtualAddress,
SECTION_ALIGNMENT)
segment_offset = align_to(opt.SizeOfHeaders - sections[0].VirtualAddress, SECTION_ALIGNMENT)
opt.AddressOfEntryPoint = file["e_entry"] + segment_offset
opt.AddressOfEntryPoint = file['e_entry'] + segment_offset
opt.BaseOfCode += segment_offset
if isinstance(opt, PeOptionalHeader32):
opt.BaseOfData += segment_offset
pe_reloc_blocks: dict[int, PeRelocationBlock] = {}
for reloc_type, reloc_table in dynamic.get_relocation_tables().items():
if reloc_type not in ["REL", "RELA"]:
raise BadSectionError(f"Unsupported relocation type {reloc_type}")
convert_elf_reloc_table(file,
reloc_table,
opt.ImageBase + segment_offset,
sections,
pe_reloc_blocks)
if reloc_type not in ['REL', 'RELA']:
raise BadSectionError(f'Unsupported relocation type {reloc_type}')
convert_elf_reloc_table(file, reloc_table, opt.ImageBase + segment_offset, sections, pe_reloc_blocks)
for pe_s in sections:
pe_s.VirtualAddress += segment_offset
@@ -524,7 +524,7 @@ def convert_elf_relocations(
data += entry
pe_reloc_s = PeSection()
pe_reloc_s.Name = b".reloc"
pe_reloc_s.Name = b'.reloc'
pe_reloc_s.data = data
pe_reloc_s.VirtualAddress = next_section_address(sections)
pe_reloc_s.VirtualSize = len(data)
@@ -543,9 +543,9 @@ def write_pe(
opt: PeOptionalHeader,
sections: list[PeSection],
) -> None:
file.write(b"MZ")
file.write(b'MZ')
file.seek(0x3C, io.SEEK_SET)
file.write(PE_OFFSET.to_bytes(2, byteorder="little"))
file.write(PE_OFFSET.to_bytes(2, byteorder='little'))
file.seek(PE_OFFSET, io.SEEK_SET)
file.write(PE_MAGIC)
file.write(coff)
@@ -554,8 +554,9 @@ def write_pe(
offset = opt.SizeOfHeaders
for pe_s in sorted(sections, key=lambda s: s.VirtualAddress):
if pe_s.VirtualAddress < opt.SizeOfHeaders:
raise BadSectionError(f"Section {pe_s.Name} @0x{pe_s.VirtualAddress:x} overlaps"
" PE headers ending at 0x{opt.SizeOfHeaders:x}")
raise BadSectionError(
f'Section {pe_s.Name} @{pe_s.VirtualAddress:#x} overlaps PE headers ending at {opt.SizeOfHeaders:#x}'
)
pe_s.PointerToRawData = offset
file.write(pe_s)
@@ -573,20 +574,20 @@ def write_pe(
def elf2efi(args: argparse.Namespace) -> None:
file = elffile.ELFFile(args.ELF)
if not file.little_endian:
raise ValueError("ELF file is not little-endian")
if file["e_type"] not in ["ET_DYN", "ET_EXEC"]:
raise ValueError(f"Unsupported ELF type {file['e_type']}")
raise ValueError('ELF file is not little-endian')
if file['e_type'] not in ['ET_DYN', 'ET_EXEC']:
raise ValueError(f'Unsupported ELF type {file["e_type"]}')
pe_arch = {
"EM_386": 0x014C,
"EM_AARCH64": 0xAA64,
"EM_ARM": 0x01C2,
"EM_LOONGARCH": 0x6232 if file.elfclass == 32 else 0x6264,
"EM_RISCV": 0x5032 if file.elfclass == 32 else 0x5064,
"EM_X86_64": 0x8664,
}.get(file["e_machine"])
'EM_386': 0x014C,
'EM_AARCH64': 0xAA64,
'EM_ARM': 0x01C2,
'EM_LOONGARCH': 0x6232 if file.elfclass == 32 else 0x6264,
'EM_RISCV': 0x5032 if file.elfclass == 32 else 0x5064,
'EM_X86_64': 0x8664,
}.get(file['e_machine']) # fmt: skip
if pe_arch is None:
raise ValueError(f"Unsupported ELF architecture {file['e_machine']}")
raise ValueError(f'Unsupported ELF architecture {file["e_machine"]}')
coff = PeCoffHeader()
opt = PeOptionalHeader32() if file.elfclass == 32 else PeOptionalHeader32Plus()
@@ -605,7 +606,7 @@ def elf2efi(args: argparse.Namespace) -> None:
coff.Machine = pe_arch
coff.NumberOfSections = len(sections)
coff.TimeDateStamp = int(os.environ.get("SOURCE_DATE_EPOCH") or time.time())
coff.TimeDateStamp = int(os.environ.get('SOURCE_DATE_EPOCH') or time.time())
coff.SizeOfOptionalHeader = sizeof(opt)
# EXECUTABLE_IMAGE|LINE_NUMS_STRIPPED|LOCAL_SYMS_STRIPPED|DEBUG_STRIPPED
# and (32BIT_MACHINE or LARGE_ADDRESS_AWARE)
@@ -632,66 +633,64 @@ def elf2efi(args: argparse.Namespace) -> None:
opt.NumberOfRvaAndSizes = N_DATA_DIRECTORY_ENTRIES
if pe_reloc_s:
opt.BaseRelocationTable = PeDataDirectory(
pe_reloc_s.VirtualAddress, pe_reloc_s.VirtualSize
)
opt.BaseRelocationTable = PeDataDirectory(pe_reloc_s.VirtualAddress, pe_reloc_s.VirtualSize)
write_pe(args.PE, coff, opt, sections)
def create_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Convert ELF binaries to PE/EFI")
parser = argparse.ArgumentParser(description='Convert ELF binaries to PE/EFI')
parser.add_argument(
"--version-major",
'--version-major',
type=int,
default=0,
help="Major image version of EFI image",
help='Major image version of EFI image',
)
parser.add_argument(
"--version-minor",
'--version-minor',
type=int,
default=0,
help="Minor image version of EFI image",
help='Minor image version of EFI image',
)
parser.add_argument(
"--efi-major",
'--efi-major',
type=int,
default=0,
help="Minimum major EFI subsystem version",
help='Minimum major EFI subsystem version',
)
parser.add_argument(
"--efi-minor",
'--efi-minor',
type=int,
default=0,
help="Minimum minor EFI subsystem version",
help='Minimum minor EFI subsystem version',
)
parser.add_argument(
"--subsystem",
'--subsystem',
type=int,
default=10,
help="PE subsystem",
help='PE subsystem',
)
parser.add_argument(
"ELF",
type=argparse.FileType("rb"),
help="Input ELF file",
'ELF',
type=argparse.FileType('rb'),
help='Input ELF file',
)
parser.add_argument(
"PE",
type=argparse.FileType("wb"),
help="Output PE/EFI file",
'PE',
type=argparse.FileType('wb'),
help='Output PE/EFI file',
)
parser.add_argument(
"--minimum-sections",
'--minimum-sections',
type=int,
default=0,
help="Minimum number of sections to leave space for",
help='Minimum number of sections to leave space for',
)
parser.add_argument(
"--copy-sections",
'--copy-sections',
type=str,
default="",
help="Copy these sections if found",
default='',
help='Copy these sections if found',
)
return parser
@@ -701,5 +700,5 @@ def main() -> None:
elf2efi(parser.parse_args())
if __name__ == "__main__":
if __name__ == '__main__':
main()
tools/fetch-distro.py
+47 -28
View File
@@ -12,6 +12,7 @@ import shlex
import subprocess
from pathlib import Path
def parse_args():
p = argparse.ArgumentParser(
description=__doc__,
@@ -27,34 +28,35 @@ def parse_args():
default=True,
)
p.add_argument(
'--update', '-u',
'--update',
'-u',
action='store_true',
default=False,
)
p.add_argument('--profile')
return p.parse_args()
def read_config(distro: str):
cmd = ['mkosi', '--json', '-d', distro, '-f', 'summary']
if args.profile:
cmd += ['--profile', args.profile]
print(f"+ {shlex.join(cmd)}")
print(f'+ {shlex.join(cmd)}')
text = subprocess.check_output(cmd, text=True)
data = json.loads(text)
images = {image["Image"]: image for image in data["Images"]}
return images["build"]
images = {image['Image']: image for image in data['Images']}
return images['build']
def commit_file(distro: str, files: list[Path], commit: str, changes: str):
message = '\n'.join((
f'mkosi: update {distro} commit reference to {commit}',
'',
changes))
message = '\n'.join((f'mkosi: update {distro} commit reference to {commit}', '', changes))
cmd = ['git', 'commit', '-m', message, *(str(file) for file in files)]
print(f"+ {shlex.join(cmd)}")
print(f'+ {shlex.join(cmd)}')
subprocess.check_call(cmd)
def checkout_distro(args, distro: str, config: dict):
url = config['Environment']['GIT_URL']
branch = config['Environment']['GIT_BRANCH']
@@ -74,28 +76,30 @@ def checkout_distro(args, distro: str, config: dict):
reference = ['--reference-if-able=.'] if distro == 'debian' else []
cmd = [
'git', 'clone', url,
'git',
'clone',
url,
f'--branch={branch}',
*sparse,
dest.as_posix(),
*reference,
]
print(f"+ {shlex.join(cmd)}")
print(f'+ {shlex.join(cmd)}')
subprocess.check_call(cmd)
# Sparse checkout if the package is in a subdirectory
if subdir is not None:
cmd = ['git', '-C', f'pkg/{pkg_subdir}', 'sparse-checkout', 'set',
'--no-cone', f'{subdir}']
print(f"+ {shlex.join(cmd)}")
cmd = ['git', '-C', f'pkg/{pkg_subdir}', 'sparse-checkout', 'set', '--no-cone', f'{subdir}']
print(f'+ {shlex.join(cmd)}')
subprocess.check_call(cmd)
cmd = ['git', '-C', f'pkg/{pkg_subdir}', 'checkout', 'HEAD']
print(f"+ {shlex.join(cmd)}")
print(f'+ {shlex.join(cmd)}')
subprocess.check_call(cmd)
args.fetch = False # no need to fetch if we just cloned
def update_distro(args, distro: str, config: dict):
branch = config['Environment']['GIT_BRANCH']
subdir = config['Environment'].get('GIT_SUBDIR')
@@ -103,32 +107,46 @@ def update_distro(args, distro: str, config: dict):
pkg_subdir = config['Environment']['PKG_SUBDIR']
if args.fetch:
cmd = ['git', '-C', f'pkg/{pkg_subdir}', 'fetch', 'origin', '-v',
f'{branch}:remotes/origin/{branch}']
print(f"+ {shlex.join(cmd)}")
cmd = [
'git',
'-C', f'pkg/{pkg_subdir}',
'fetch',
'origin',
'-v',
f'{branch}:remotes/origin/{branch}',
] # fmt: skip
print(f'+ {shlex.join(cmd)}')
subprocess.check_call(cmd)
cmd = ['git', '-C', f'pkg/{pkg_subdir}', 'switch', branch]
print(f"+ {shlex.join(cmd)}")
print(f'+ {shlex.join(cmd)}')
subprocess.check_call(cmd)
cmd = ['git', '-C', f'pkg/{pkg_subdir}', 'log', '-n1', '--format=%H',
f'refs/remotes/origin/{branch}']
cmd = ['git', '-C', f'pkg/{pkg_subdir}', 'log', '-n1', '--format=%H', f'refs/remotes/origin/{branch}']
if subdir is not None:
cmd += [f'{subdir}']
print(f"+ {shlex.join(cmd)}")
print(f'+ {shlex.join(cmd)}')
new_commit = subprocess.check_output(cmd, text=True).strip()
if old_commit == new_commit:
print(f'{pkg_subdir}: commit {new_commit!s} is still fresh')
return
cmd = ['git', '-C', f'pkg/{pkg_subdir}', 'log', '--graph', '--no-merges',
'--pretty=oneline', '--no-decorate', '--abbrev-commit', '--abbrev=10',
f'{old_commit}..{new_commit}']
cmd = [
'git',
'-C', f'pkg/{pkg_subdir}',
'log',
'--graph',
'--no-merges',
'--pretty=oneline',
'--no-decorate',
'--abbrev-commit',
'--abbrev=10',
f'{old_commit}..{new_commit}',
] # fmt: skip
if subdir is not None:
cmd += [f'{subdir}']
print(f"+ {shlex.join(cmd)}")
print(f'+ {shlex.join(cmd)}')
changes = subprocess.check_output(cmd, text=True).strip()
conf_dir = Path('mkosi/mkosi.pkgenv/mkosi.conf.d')
@@ -142,8 +160,8 @@ def update_distro(args, distro: str, config: dict):
file.write_text(new)
tocommit = [file]
if distro == "fedora":
packit = Path(".packit.yml")
if distro == 'fedora':
packit = Path('.packit.yml')
s = packit.read_text()
assert old_commit in s
new = s.replace(old_commit, new_commit)
@@ -155,6 +173,7 @@ def update_distro(args, distro: str, config: dict):
else:
raise ValueError(f'{distro}: hash {new_commit} not found under {conf_dir}')
if __name__ == '__main__':
args = parse_args()
tools/fetch-mkosi.py
+33 -18
View File
@@ -7,9 +7,9 @@ With -u, if changed, commit the latest hash.
"""
import argparse
import re
import shlex
import subprocess
import re
from pathlib import Path
URL = 'https://github.com/systemd/mkosi'
@@ -17,6 +17,7 @@ BRANCH = 'main' # We only want to ever use commits on upstream 'main' branch
CONFIG = Path('mkosi/mkosi.conf')
WORKFLOWS = [Path('.github/workflows') / f for f in ['mkosi.yml', 'coverage.yml', 'linter.yml']]
def parse_args():
p = argparse.ArgumentParser(
description=__doc__,
@@ -26,59 +27,72 @@ def parse_args():
type=Path,
)
p.add_argument(
'--update', '-u',
'--update',
'-u',
action='store_true',
default=False,
)
return p.parse_args()
def read_config():
print(f'Reading {CONFIG}…')
matches = [m.group(1)
for line in open(CONFIG)
if (m := re.match('^MinimumVersion=commit:([a-z0-9]{40})$',
line.strip()))]
c = CONFIG.read_text()
matches = [
m.group(1) for m in re.finditer('^\s*MinimumVersion=commit:([a-z0-9]{40})\s*$', c, re.MULTILINE)
]
assert len(matches) == 1
return matches[0]
def commit_file(files: list[Path], commit: str, changes: str):
message = '\n'.join((
f'mkosi: update mkosi ref to {commit}',
'',
changes))
message = '\n'.join((f'mkosi: update mkosi ref to {commit}', '', changes))
cmd = ['git', 'commit', '-m', message, *(str(file) for file in files)]
print(f"+ {shlex.join(cmd)}")
print(f'+ {shlex.join(cmd)}')
subprocess.check_call(cmd)
def checkout_mkosi(args):
if args.dir.exists():
print(f'{args.dir} already exists.')
return
cmd = [
'git', 'clone', URL,
'git',
'clone',
URL,
f'--branch={BRANCH}',
args.dir.as_posix(),
]
print(f"+ {shlex.join(cmd)}")
print(f'+ {shlex.join(cmd)}')
subprocess.check_call(cmd)
def update_mkosi(args):
old_commit = read_config()
cmd = ['git', '-C', args.dir.as_posix(), 'rev-parse', f'refs/remotes/origin/{BRANCH}']
print(f"+ {shlex.join(cmd)}")
print(f'+ {shlex.join(cmd)}')
new_commit = subprocess.check_output(cmd, text=True).strip()
if old_commit == new_commit:
print(f'mkosi: commit {new_commit!s} is still fresh')
return
cmd = ['git', '-C', args.dir.as_posix(), 'log', '--graph', '--no-merges',
'--pretty=oneline', '--no-decorate', '--abbrev-commit', '--abbrev=10',
f'{old_commit}..{new_commit}']
print(f"+ {shlex.join(cmd)}")
cmd = [
'git',
'-C', args.dir.as_posix(),
'log',
'--graph',
'--no-merges',
'--pretty=oneline',
'--no-decorate',
'--abbrev-commit',
'--abbrev=10',
f'{old_commit}..{new_commit}',
] # fmt: skip
print(f'+ {shlex.join(cmd)}')
changes = subprocess.check_output(cmd, text=True).strip()
for f in [CONFIG, *WORKFLOWS]:
@@ -91,6 +105,7 @@ def update_mkosi(args):
commit_file([CONFIG, *WORKFLOWS], new_commit, changes)
if __name__ == '__main__':
args = parse_args()
checkout_mkosi(args)
tools/find-unused-library-symbols.py
+37 -38
View File
@@ -39,10 +39,10 @@ def get_exported_symbols(library_path):
['nm', '--dynamic', '--defined-only', '--extern-only', library_path],
capture_output=True,
text=True,
check=True
check=True,
)
except subprocess.CalledProcessError as e:
print(f"Error: Failed to run nm on {library_path}: {e}", file=sys.stderr)
print(f'Error: Failed to run nm on {library_path}: {e}', file=sys.stderr)
sys.exit(1)
except FileNotFoundError:
print("Error: 'nm' command not found. Please install binutils.", file=sys.stderr)
@@ -79,10 +79,10 @@ def get_undefined_symbols(executable_path):
['nm', '--dynamic', '--undefined-only', executable_path],
capture_output=True,
text=True,
check=True
check=True,
)
except subprocess.CalledProcessError as e:
print(f"Warning: Failed to run nm on {executable_path}: {e}", file=sys.stderr)
print(f'Warning: Failed to run nm on {executable_path}: {e}', file=sys.stderr)
return set()
except FileNotFoundError:
print("Error: 'nm' command not found. Please install binutils.", file=sys.stderr)
@@ -112,7 +112,7 @@ def verify_executable_links_library(executable_path, library_name):
['ldd', executable_path],
capture_output=True,
text=True,
check=True
check=True,
)
except (subprocess.CalledProcessError, FileNotFoundError):
# If ldd fails or doesn't exist, we'll skip the verification
@@ -137,14 +137,16 @@ def get_library_internal_references(library_path, exported_symbols):
['objdump', '-R', library_path],
capture_output=True,
text=True,
check=True
check=True,
)
except subprocess.CalledProcessError as e:
print(f"Warning: Failed to run objdump on {library_path}: {e}", file=sys.stderr)
print(f'Warning: Failed to run objdump on {library_path}: {e}', file=sys.stderr)
return set()
except FileNotFoundError:
print("Warning: 'objdump' command not found. Internal references won't be detected.",
file=sys.stderr)
print(
"Warning: 'objdump' command not found. Internal references won't be detected.",
file=sys.stderr,
)
return set()
internal_refs = set()
@@ -181,7 +183,7 @@ def find_unused_symbols(library_path, executable_paths, verify_linkage=True):
exported_symbols = get_exported_symbols(library_path)
if not exported_symbols:
print(f"Warning: No exported symbols found in {library_path}", file=sys.stderr)
print(f'Warning: No exported symbols found in {library_path}', file=sys.stderr)
return set(), set(), set()
# Collect all symbols used by the executables
@@ -194,8 +196,7 @@ def find_unused_symbols(library_path, executable_paths, verify_linkage=True):
for exe_path in executable_paths:
# Optionally verify linkage
if verify_linkage and not verify_executable_links_library(exe_path, library_name):
print(f"Warning: {exe_path} does not appear to link against {library_name}",
file=sys.stderr)
print(f'Warning: {exe_path} does not appear to link against {library_name}', file=sys.stderr)
undefined_symbols = get_undefined_symbols(exe_path)
# Only count symbols that are actually exported by our library
@@ -209,31 +210,31 @@ def find_unused_symbols(library_path, executable_paths, verify_linkage=True):
def main():
parser = argparse.ArgumentParser(
description='Find unused exported symbols in a shared library'
description='Find unused exported symbols in a shared library',
)
parser.add_argument(
'library',
help='Path to the shared library to analyze'
help='Path to the shared library to analyze',
)
parser.add_argument(
'executables',
nargs='+',
help='Paths to executables that link against the library'
help='Paths to executables that link against the library',
)
parser.add_argument(
'--no-verify-linkage',
action='store_true',
help='Skip verification that executables actually link against the library'
help='Skip verification that executables actually link against the library',
)
parser.add_argument(
'--show-used',
action='store_true',
help='Also show used symbols'
help='Also show used symbols',
)
parser.add_argument(
'--stats-only',
action='store_true',
help='Only show statistics, not individual symbols'
help='Only show statistics, not individual symbols',
)
args = parser.parse_args()
@@ -241,7 +242,7 @@ def main():
# Verify library exists
library_path = Path(args.library)
if not library_path.exists():
print(f"Error: Library not found: {library_path}", file=sys.stderr)
print(f'Error: Library not found: {library_path}', file=sys.stderr)
sys.exit(1)
# Verify executables exist
@@ -249,47 +250,45 @@ def main():
for exe in args.executables:
exe_path = Path(exe)
if not exe_path.exists():
print(f"Warning: Executable not found: {exe_path}", file=sys.stderr)
print(f'Warning: Executable not found: {exe_path}', file=sys.stderr)
else:
executable_paths.append(str(exe_path))
if not executable_paths:
print("Error: No valid executables provided", file=sys.stderr)
print('Error: No valid executables provided', file=sys.stderr)
sys.exit(1)
# Analyze symbols
unused, exported, used = find_unused_symbols(
str(library_path),
executable_paths,
verify_linkage=not args.no_verify_linkage
str(library_path), executable_paths, verify_linkage=not args.no_verify_linkage
)
# Print results
print(f"Analysis of {library_path.name}")
print("=" * 70)
print(f"Total exported symbols: {len(exported)}")
print(f" (excluding public API symbols starting with 'sd_')")
print(f"Used symbols: {len(used)}")
print(f"Unused symbols: {len(unused)}")
print(f"Usage rate: {len(used)/len(exported)*100:.1f}%" if exported else "N/A")
print(f'Analysis of {library_path.name}')
print('=' * 70)
print(f'Total exported symbols: {len(exported)}')
print(" (excluding public API symbols starting with 'sd_')")
print(f'Used symbols: {len(used)}')
print(f'Unused symbols: {len(unused)}')
print(f'Usage rate: {len(used) / len(exported) * 100:.1f}%' if exported else 'N/A')
print()
if not args.stats_only:
if unused:
print("Unused symbols:")
print("-" * 70)
print('Unused symbols:')
print('-' * 70)
for symbol in sorted(unused):
print(f" {symbol}")
print(f' {symbol}')
print()
else:
print("All exported symbols are used!")
print('All exported symbols are used!')
print()
if args.show_used and used:
print("Used symbols:")
print("-" * 70)
print('Used symbols:')
print('-' * 70)
for symbol in sorted(used):
print(f" {symbol}")
print(f' {symbol}')
print()
# Exit with non-zero if there are unused symbols (useful for CI)
tools/gdb-sd_dump_hashmaps.py
+29 -27
View File
@@ -4,38 +4,39 @@
import gdb
class sd_dump_hashmaps(gdb.Command):
"dump systemd's hashmaps"
def __init__(self):
super().__init__("sd_dump_hashmaps", gdb.COMMAND_DATA, gdb.COMPLETE_NONE)
super().__init__('sd_dump_hashmaps', gdb.COMMAND_DATA, gdb.COMPLETE_NONE)
def invoke(self, arg, _from_tty):
d = gdb.parse_and_eval("hashmap_debug_list")
hashmap_type_info = gdb.parse_and_eval("hashmap_type_info")
uchar_t = gdb.lookup_type("unsigned char")
ulong_t = gdb.lookup_type("unsigned long")
debug_offset = gdb.parse_and_eval("(unsigned long)&((HashmapBase*)0)->debug")
d = gdb.parse_and_eval('hashmap_debug_list')
hashmap_type_info = gdb.parse_and_eval('hashmap_type_info')
uchar_t = gdb.lookup_type('unsigned char')
ulong_t = gdb.lookup_type('unsigned long')
debug_offset = gdb.parse_and_eval('(unsigned long)&((HashmapBase*)0)->debug')
print("type, hash, indirect, entries, max_entries, buckets, creator")
print('type, hash, indirect, entries, max_entries, buckets, creator')
while d:
h = gdb.parse_and_eval(f"(HashmapBase*)((char*){int(d.cast(ulong_t))} - {debug_offset})")
h = gdb.parse_and_eval(f'(HashmapBase*)((char*){int(d.cast(ulong_t))} - {debug_offset})')
if h["has_indirect"]:
storage_ptr = h["indirect"]["storage"].cast(uchar_t.pointer())
n_entries = h["indirect"]["n_entries"]
n_buckets = h["indirect"]["n_buckets"]
if h['has_indirect']:
storage_ptr = h['indirect']['storage'].cast(uchar_t.pointer())
n_entries = h['indirect']['n_entries']
n_buckets = h['indirect']['n_buckets']
else:
storage_ptr = h["direct"]["storage"].cast(uchar_t.pointer())
n_entries = h["n_direct_entries"]
n_buckets = hashmap_type_info[h["type"]]["n_direct_buckets"]
storage_ptr = h['direct']['storage'].cast(uchar_t.pointer())
n_entries = h['n_direct_entries']
n_buckets = hashmap_type_info[h['type']]['n_direct_buckets']
t = ["plain", "ordered", "set"][int(h["type"])]
t = ['plain', 'ordered', 'set'][int(h['type'])]
print(f'{t}, {h["hash_ops"]}, {bool(h["has_indirect"])}, {n_entries}, {d["max_entries"]}, {n_buckets}')
print(f'{t}, {h["hash_ops"]}, {bool(h["has_indirect"])}, {n_entries}, {d["max_entries"]}, {n_buckets}') # fmt: skip
if arg != "" and n_entries > 0:
dib_raw_addr = storage_ptr + hashmap_type_info[h["type"]]["entry_size"] * n_buckets
if arg != '' and n_entries > 0:
dib_raw_addr = storage_ptr + hashmap_type_info[h['type']]['entry_size'] * n_buckets
histogram = {}
for i in range(0, n_buckets):
@@ -44,11 +45,11 @@ class sd_dump_hashmaps(gdb.Command):
for dib in sorted(histogram):
if dib != 255:
print(f"{dib:>3} {histogram[dib]:>8} {float(histogram[dib]/n_entries):.0%} of entries")
print(f'{dib:>3} {histogram[dib]:>8} {float(histogram[dib] / n_entries):.0%} of entries') # fmt: skip
else:
print(f"{dib:>3} {histogram[dib]:>8} {float(histogram[dib]/n_buckets):.0%} of slots")
s = sum(dib*count for (dib, count) in histogram.items() if dib != 255) / n_entries
print(f"mean DIB of entries: {s}")
print(f'{dib:>3} {histogram[dib]:>8} {float(histogram[dib] / n_buckets):.0%} of slots') # fmt: skip
s = sum(dib * count for (dib, count) in histogram.items() if dib != 255) / n_entries
print(f'mean DIB of entries: {s}')
blocks = []
current_len = 1
@@ -69,10 +70,11 @@ class sd_dump_hashmaps(gdb.Command):
if len(blocks) > 1 and blocks[0][0] == blocks[0][1] and blocks[-1][0] == n_buckets - 1:
blocks[0][1] += blocks[-1][1]
blocks = blocks[0:-1]
print("max block: {}".format(max(blocks, key=lambda a: a[1])))
print("sum block lens: {}".format(sum(b[1] for b in blocks)))
print("mean block len: {}".format(sum(b[1] for b in blocks) / len(blocks)))
print(f'max block: {max(blocks, key=lambda a: a[1])}')
print(f'sum block lens: {sum(b[1] for b in blocks)}')
print(f'mean block len: {sum(b[1] for b in blocks) / len(blocks)}')
d = d['debug_list_next']
d = d["debug_list_next"]
sd_dump_hashmaps()
tools/generate-gperfs.py
+6 -6
View File
@@ -13,23 +13,23 @@ if __name__ == '__main__':
sys.exit(f'Usage: {sys.argv[0]} name prefix file [includes...]')
name, prefix, file, *includes = sys.argv[1:]
includes = [f"#include {i}" for i in includes]
includes = [f'#include {i}' for i in includes]
# Older versions of python don't allow backslashes
# in f-strings so use chr(10) for newlines and chr(92)
# for backslashes instead as a workaround.
print(f"""\
print(f'''\
%{{
_Pragma("GCC diagnostic ignored {chr(92)}"-Wimplicit-fallthrough{chr(92)}"")
#if __GNUC__ >= 15
_Pragma("GCC diagnostic ignored {chr(92)}"-Wzero-as-null-pointer-constant{chr(92)}"")
#endif
{chr(10).join(includes)}
%}}""")
print(f"""\
%}}''')
print(f'''\
struct {name}_name {{ const char* name; int id; }};
%null-strings
%%""")
%%''')
for line in open(file):
print("{0}, {1}{0}".format(line.rstrip(), prefix))
print('{0}, {1}{0}'.format(line.rstrip(), prefix))
tools/list-discoverable-partitions.py
+45 -26
View File
@@ -30,16 +30,15 @@ ARCHITECTURES = {
'TILEGX': 'TILE-Gx',
'X86': 'x86',
'X86_64': 'amd64/x86_64',
}
} # fmt: skip
TYPES = {
'ROOT' : 'Root Partition',
'ROOT_VERITY' : 'Root Verity Partition',
'ROOT_VERITY_SIG' : 'Root Verity Signature Partition',
'USR' : '`/usr/` Partition',
'USR_VERITY' : '`/usr/` Verity Partition',
'USR_VERITY_SIG' : '`/usr/` Verity Signature Partition',
'ROOT': 'Root Partition',
'ROOT_VERITY': 'Root Verity Partition',
'ROOT_VERITY_SIG': 'Root Verity Signature Partition',
'USR': '`/usr/` Partition',
'USR_VERITY': '`/usr/` Verity Partition',
'USR_VERITY_SIG': '`/usr/` Verity Signature Partition',
'ESP': 'EFI System Partition',
'SRV': 'Server Data Partition',
'VAR': 'Variable Data Partition',
@@ -49,7 +48,7 @@ TYPES = {
'USER_HOME': 'Per-user Home Partition',
'LINUX_GENERIC': 'Generic Linux Data Partition',
'XBOOTLDR': 'Extended Boot Loader Partition',
}
} # fmt: skip
DESCRIPTIONS = {
'ROOT': (
@@ -57,56 +56,66 @@ DESCRIPTIONS = {
'On systems with matching architecture, the first partition with this type UUID on the disk '
'containing the active EFI ESP is automatically mounted to the root directory `/`. '
'If the partition is encrypted with LUKS or has dm-verity integrity data (see below), the '
'device mapper file will be named `/dev/mapper/root`.'),
'device mapper file will be named `/dev/mapper/root`.',
),
'USR': (
'Any native, optionally in LUKS',
'Similar semantics to root partition, but just the `/usr/` partition.'),
'Similar semantics to root partition, but just the `/usr/` partition.',
),
'ROOT_VERITY': (
'A dm-verity superblock followed by hash data',
'Contains dm-verity integrity hash data for the matching root partition. If this feature is '
'used the partition UUID of the root partition should be the first 128 bits of the root hash '
'of the dm-verity hash data, and the partition UUID of this dm-verity partition should be the '
'final 128 bits of it, so that the root partition and its Verity partition can be discovered '
'easily, simply by specifying the root hash.'),
'easily, simply by specifying the root hash.',
),
'USR_VERITY': (
'A dm-verity superblock followed by hash data',
'Similar semantics to root Verity partition, but just for the `/usr/` partition.'),
'Similar semantics to root Verity partition, but just for the `/usr/` partition.',
),
'ROOT_VERITY_SIG': (
'A serialized JSON object, see below',
'Contains a root hash and a PKCS#7 signature for it, permitting signed dm-verity GPT images.'),
'Contains a root hash and a PKCS#7 signature for it, permitting signed dm-verity GPT images.',
),
'USR_VERITY_SIG': (
'A serialized JSON object, see below',
'Similar semantics to root Verity signature partition, but just for the `/usr/` partition.'),
'Similar semantics to root Verity signature partition, but just for the `/usr/` partition.',
),
'ESP': (
'VFAT',
'The ESP used for the current boot is automatically mounted to `/efi/` (or `/boot/` as '
'fallback), unless a different partition is mounted there (possibly via `/etc/fstab`, or '
'because the Extended Boot Loader Partition — see below — exists) or the directory is '
'non-empty on the root disk. This partition type is defined by the '
'[UEFI Specification](http://www.uefi.org/specifications).'),
'[UEFI Specification](http://www.uefi.org/specifications).',
),
'XBOOTLDR': (
'Typically VFAT',
'The Extended Boot Loader Partition (XBOOTLDR) used for the current boot is automatically '
'mounted to `/boot/`, unless a different partition is mounted there (possibly via '
'`/etc/fstab`) or the directory is non-empty on the root disk. This partition type '
'is defined by the [Boot Loader '
'Specification](https://uapi-group.org/specifications/specs/boot_loader_specification).'),
'Specification](https://uapi-group.org/specifications/specs/boot_loader_specification).',
),
'SWAP': (
'Swap, optionally in LUKS',
'All swap partitions on the disk containing the root partition are automatically enabled. '
'If the partition is encrypted with LUKS, the device mapper file will be named '
'`/dev/mapper/swap`. This partition type predates the Discoverable Partitions Specification.'),
'`/dev/mapper/swap`. This partition type predates the Discoverable Partitions Specification.',
),
'HOME': (
'Any native, optionally in LUKS',
'The first partition with this type UUID on the disk containing the root partition is '
'automatically mounted to `/home/`. If the partition is encrypted with LUKS, the device '
'mapper file will be named `/dev/mapper/home`.'),
'mapper file will be named `/dev/mapper/home`.',
),
'SRV': (
'Any native, optionally in LUKS',
'The first partition with this type UUID on the disk containing the root partition is '
'automatically mounted to `/srv/`. If the partition is encrypted with LUKS, the device '
'mapper file will be named `/dev/mapper/srv`.'),
'mapper file will be named `/dev/mapper/srv`.',
),
'VAR': (
'Any native, optionally in LUKS',
'The first partition with this type UUID on the disk containing the root partition is '
@@ -118,7 +127,8 @@ DESCRIPTIONS = {
'listed here) is inherently private to a specific installation and cannot possibly be '
'shared between multiple OS installations on the same disk, and thus should be bound to '
'a specific instance of the OS, identified by its machine ID. If the partition is '
'encrypted with LUKS, the device mapper file will be named `/dev/mapper/var`.'),
'encrypted with LUKS, the device mapper file will be named `/dev/mapper/var`.',
),
'TMP': (
'Any native, optionally in LUKS',
'The first partition with this type UUID on the disk containing the root partition is '
@@ -127,19 +137,23 @@ DESCRIPTIONS = {
'is indeed `/var/tmp/`, not `/tmp/`. The latter is typically maintained in memory via '
'`tmpfs` and does not require a partition on disk. In some cases it might be '
'desirable to make `/tmp/` persistent too, in which case it is recommended to make it '
'a symlink or bind mount to `/var/tmp/`, thus not requiring its own partition type UUID.'),
'a symlink or bind mount to `/var/tmp/`, thus not requiring its own partition type UUID.',
),
'USER_HOME': (
'Any native, optionally in LUKS',
'A home partition of a user, managed by '
'[`systemd-homed`](https://www.freedesktop.org/software/systemd/man/systemd-homed.html).'),
'[`systemd-homed`](https://www.freedesktop.org/software/systemd/man/systemd-homed.html).',
),
'LINUX_GENERIC': (
'Any native, optionally in LUKS',
'No automatic mounting takes place for other Linux data partitions. This partition type '
'should be used for all partitions that carry Linux file systems. The installer needs '
'to mount them explicitly via entries in `/etc/fstab`. Optionally, these partitions may '
'be encrypted with LUKS. This partition type predates the Discoverable Partitions Specification.'),
'be encrypted with LUKS. This partition type predates the Discoverable Partitions Specification.',
),
}
def extract(file):
for line in file:
# print(line)
@@ -148,7 +162,10 @@ def extract(file):
continue
name = line.split()[1]
if m2 := re.match(r'^(ROOT|USR)_([A-Z0-9]+|X86_64|PPC64_LE|MIPS_LE|MIPS64_LE)(|_VERITY|_VERITY_SIG)\s+SD_ID128_MAKE\((.*)\)', m.group(1)):
if m2 := re.match(
r'^(ROOT|USR)_([A-Z0-9]+|X86_64|PPC64_LE|MIPS_LE|MIPS64_LE)(|_VERITY|_VERITY_SIG)\s+SD_ID128_MAKE\((.*)\)',
m.group(1),
):
ptype, arch, suffix, u = m2.groups()
u = uuid.UUID(u.replace(',', ''))
assert arch in ARCHITECTURES, f'{arch} not in f{ARCHITECTURES}'
@@ -165,6 +182,7 @@ def extract(file):
else:
raise ValueError(f'Failed to match: {m.group(1)}')
def generate(defines):
prevtype = None
@@ -188,6 +206,7 @@ def generate(defines):
print(f'| _{tdesc}{adesc}_ | `{puuid}` `{name}` | {morea} | {moreb} |')
if __name__ == '__main__':
known = extract(sys.stdin)
generate(known)
tools/make-directive-index.py
+22 -16
View File
@@ -13,6 +13,7 @@ This index contains {count} entries in {sections} sections,
referring to {pages} individual manual pages.
'''
def _extract_directives(directive_groups, formatting, page):
t = xml_parse(page)
section = t.find('./refmeta/manvolnum').text
@@ -21,12 +22,13 @@ def _extract_directives(directive_groups, formatting, page):
storopt = directive_groups['options']
for variablelist in t.iterfind('.//variablelist'):
klass = variablelist.attrib.get('class')
searchpath = variablelist.attrib.get('xpath','./varlistentry/term/varname')
searchpath = variablelist.attrib.get('xpath', './varlistentry/term/varname')
storvar = directive_groups[klass or 'miscellaneous']
# <option>s go in OPTIONS, unless class is specified
for xpath, stor in ((searchpath, storvar),
('./varlistentry/term/option',
storvar if klass else storopt)):
for xpath, stor in (
(searchpath, storvar),
('./varlistentry/term/option', storvar if klass else storopt),
):
for name in variablelist.iterfind(xpath):
text = re.sub(r'([= ]).*', r'\1', name.text).rstrip()
if text.startswith('-'):
@@ -45,14 +47,16 @@ def _extract_directives(directive_groups, formatting, page):
if extra:
stor[extra].append((pagename, section))
if extra not in formatting:
elt = tree.Element("varname")
elt.text= extra
elt = tree.Element('varname')
elt.text = extra
formatting[extra] = elt
storfile = directive_groups['filenames']
for xpath, absolute_only in (('.//refsynopsisdiv//filename', False),
('.//refsynopsisdiv//command', False),
('.//filename', True)):
for xpath, absolute_only in (
('.//refsynopsisdiv//filename', False),
('.//refsynopsisdiv//command', False),
('.//filename', True),
):
for name in t.iterfind(xpath):
if absolute_only and not (name.text and name.text.startswith('/')):
continue
@@ -82,7 +86,7 @@ def _extract_directives(directive_groups, formatting, page):
if name.attrib.get('index') == 'false':
continue
name.tail = ''
if name.text.startswith('('): # a cast, strip it
if name.text.startswith('('): # a cast, strip it
name.text = name.text.partition(' ')[2]
klass = name.attrib.get('class') or 'constants'
storfile = directive_groups[klass]
@@ -101,6 +105,7 @@ def _extract_directives(directive_groups, formatting, page):
storfile[name.text].append((pagename, section))
formatting[name.text] = name
def _make_section(template, name, directives, formatting):
varlist = template.find(f".//*[@id='{name}']")
for varname, manpages in sorted(directives.items()):
@@ -123,6 +128,7 @@ def _make_section(template, name, directives, formatting):
d.text = manvolume
entry.tail = '\n\n'
def _make_colophon(template, groups):
count = 0
pages = set()
@@ -132,9 +138,8 @@ def _make_colophon(template, groups):
pages |= set(pagelist)
para = template.find(".//para[@id='colophon']")
para.text = COLOPHON.format(count=count,
sections=len(groups),
pages=len(pages))
para.text = COLOPHON.format(count=count, sections=len(groups), pages=len(pages))
def _make_page(template, directive_groups, formatting):
"""Create an XML tree from directive_groups.
@@ -152,21 +157,22 @@ def _make_page(template, directive_groups, formatting):
return template
def make_page(template_path, xml_files):
"Extract directives from xml_files and return XML index tree."
template = xml_parse(template_path)
names = [vl.get('id') for vl in template.iterfind('.//variablelist')]
directive_groups = {name:collections.defaultdict(list)
for name in names}
directive_groups = {name: collections.defaultdict(list) for name in names}
formatting = {}
for page in xml_files:
try:
_extract_directives(directive_groups, formatting, page)
except Exception as e:
raise ValueError("failed to process " + page) from e
raise ValueError('failed to process ' + page) from e
return _make_page(template, directive_groups, formatting)
if __name__ == '__main__':
with open(sys.argv[1], 'wb') as f:
_template_path = sys.argv[2]
tools/make-man-index.py
+6 -1
View File
@@ -49,6 +49,7 @@ def check_id(page, t):
if not re.search('/' + page_id + '[.]', page.translate(str.maketrans('@', '_'))):
raise ValueError(f"id='{page_id}' is not the same as page name '{page}'")
def make_index(pages):
index = collections.defaultdict(list)
for p in pages:
@@ -63,6 +64,7 @@ def make_index(pages):
index[f.text[0].upper()].append(infos)
return index
def add_letter(template, letter, pages):
refsect1 = tree.SubElement(template, 'refsect1')
title = tree.SubElement(refsect1, 'title')
@@ -77,10 +79,11 @@ def add_letter(template, letter, pages):
d = tree.SubElement(b, 'manvolnum')
d.text = section
b.tail = MDASH + purpose # + ' (' + p + ')'
b.tail = MDASH + purpose # + ' (' + p + ')'
tree.SubElement(para, 'sbr')
def add_summary(template, indexpages):
count = 0
pages = set()
@@ -96,6 +99,7 @@ def add_summary(template, indexpages):
para = template.find(".//para[@id='counts']")
para.text = COUNTS.format(count=count, pages=len(pages))
def make_page(*xml_files):
template = tree.fromstring(TEMPLATE)
index = make_index(xml_files)
@@ -107,6 +111,7 @@ def make_page(*xml_files):
return template
if __name__ == '__main__':
with open(sys.argv[1], 'wb') as file:
file.write(xml_print(make_page(*sys.argv[2:])))
tools/meson-extract-unit-files.py
+9 -4
View File
@@ -6,15 +6,20 @@ import re
import subprocess
import sys
def main():
build_dir = sys.argv[1]
out = subprocess.run(["meson", "introspect", "--installed", build_dir],
stdout=subprocess.PIPE, check=True)
out = subprocess.run(
['meson', 'introspect', '--installed', build_dir],
stdout=subprocess.PIPE,
check=True,
)
files = json.loads(out.stdout)
for file in sorted(files.values()):
if re.search("^/usr/lib/systemd/(system|user)/", file) and not file.endswith(".conf"):
if re.search('^/usr/lib/systemd/(system|user)/', file) and not file.endswith('.conf'):
print(file)
if __name__ == "__main__":
if __name__ == '__main__':
main()
tools/meson-render-jinja2.py
+11 -5
View File
@@ -9,6 +9,7 @@ import sys
import jinja2
def parse_config_h(filename):
# Parse config.h file generated by meson.
ans = {}
@@ -24,15 +25,19 @@ def parse_config_h(filename):
ans[a] = b
return ans
def render(filename, defines):
text = open(filename).read()
template = jinja2.Template(text,
trim_blocks=True,
lstrip_blocks=True,
keep_trailing_newline=True,
undefined=jinja2.StrictUndefined)
template = jinja2.Template(
text,
trim_blocks=True,
lstrip_blocks=True,
keep_trailing_newline=True,
undefined=jinja2.StrictUndefined,
)
return template.render(defines)
def main():
defines = parse_config_h(sys.argv[1])
output = render(sys.argv[2], defines)
@@ -41,5 +46,6 @@ def main():
info = os.stat(sys.argv[2])
os.chmod(sys.argv[3], info.st_mode)
if __name__ == '__main__':
main()
tools/sync-docs.py
+43 -43
View File
@@ -1,21 +1,21 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: LGPL-2.1-or-later
from argparse import ArgumentParser
import glob
import json
import os
import re
import subprocess
import sys
from argparse import ArgumentParser
import requests
BASE_URL = "https://www.freedesktop.org/software/systemd/man/"
JQUERY_URL = "https://code.jquery.com/jquery-3.7.1.min.js"
BASE_URL = 'https://www.freedesktop.org/software/systemd/man/'
JQUERY_URL = 'https://code.jquery.com/jquery-3.7.1.min.js'
SCRIPT_TAG = '<script src="{}"></script>'
NAV_JS = """
NAV_JS = '''
$(document).ready(function() {
$.getJSON("../index.json", function(data) {
data.sort().reverse();
@@ -48,77 +48,77 @@ $(document).ready(function() {
});
});
});
"""
'''
def process_file(filename):
with open(filename) as f:
contents = f.read()
if SCRIPT_TAG.format("../nav.js") in contents:
if SCRIPT_TAG.format('../nav.js') in contents:
return
body_tag = re.search("<body[^>]*>", contents)
body_tag = re.search('<body[^>]*>', contents)
new_contents = (
contents[: body_tag.end()]
+ SCRIPT_TAG.format(JQUERY_URL)
+ SCRIPT_TAG.format("../nav.js")
+ SCRIPT_TAG.format('../nav.js')
+ contents[body_tag.end() :]
)
with open(filename, "w") as f:
with open(filename, 'w') as f:
f.write(new_contents)
def update_index_file(version, index_filename):
response = requests.get(BASE_URL + "index.json")
response = requests.get(BASE_URL + 'index.json')
if response.status_code == 404:
index = []
elif response.ok:
index = response.json()
else:
sys.exit(f"Error getting index: {response.status_code} {response.reason}")
sys.exit(f'Error getting index: {response.status_code} {response.reason}')
if version not in index:
index.insert(0, version)
with open(index_filename, "w") as f:
with open(index_filename, 'w') as f:
json.dump(index, f)
def get_latest_version():
tags = subprocess.check_output(["git", "tag", "-l", "v*"], text=True).split()
tags = subprocess.check_output(['git', 'tag', '-l', 'v*'], text=True).split()
versions = []
for tag in tags:
m = re.match(r"v?(\d+).*", tag)
m = re.match(r'v?(\d+).*', tag)
if m:
versions.append(int(m.group(1)))
return max(versions)
def main(version, directory, www_target):
index_filename = os.path.join(directory, "index.json")
nav_filename = os.path.join(directory, "nav.js")
index_filename = os.path.join(directory, 'index.json')
nav_filename = os.path.join(directory, 'nav.js')
# The upload directory does not contain point release suffixes
version = re.sub(r"\..+$", "", version)
version = re.sub(r'\..+$', '', version)
current_branch = subprocess.check_output(["git", "branch", "--show-current"], text=True).strip()
current_branch = subprocess.check_output(['git', 'branch', '--show-current'], text=True).strip()
if current_branch != 'main' and not current_branch.endswith("-stable"):
sys.exit("doc-sync should only be run from main or a stable branch")
if current_branch != 'main' and not current_branch.endswith('-stable'):
sys.exit('doc-sync should only be run from main or a stable branch')
for filename in glob.glob(os.path.join(directory, "*.html")):
for filename in glob.glob(os.path.join(directory, '*.html')):
process_file(filename)
if current_branch == "main":
version = "devel"
dirs = ["devel"]
if current_branch == 'main':
version = 'devel'
dirs = ['devel']
elif int(version) == get_latest_version():
dirs = [version, "latest"]
dirs = [version, 'latest']
else:
dirs = [version]
with open(nav_filename, "w") as f:
with open(nav_filename, 'w') as f:
f.write(NAV_JS)
update_index_file(version, index_filename)
@@ -126,33 +126,33 @@ def main(version, directory, www_target):
for d in dirs:
subprocess.check_call(
[
"rsync",
"-rlv",
"--delete-excluded",
"--include=*.html",
"--exclude=*",
"--omit-dir-times",
directory + "/", # copy contents of directory
os.path.join(www_target, "man", d),
'rsync',
'-rlv',
'--delete-excluded',
'--include=*.html',
'--exclude=*',
'--omit-dir-times',
directory + '/', # copy contents of directory
os.path.join(www_target, 'man', d),
]
)
subprocess.check_call(
[
"rsync",
"-v",
os.path.join(directory, "index.json"),
os.path.join(directory, "nav.js"),
os.path.join(www_target, "man"),
'rsync',
'-v',
os.path.join(directory, 'index.json'),
os.path.join(directory, 'nav.js'),
os.path.join(www_target, 'man'),
]
)
if __name__ == "__main__":
if __name__ == '__main__':
parser = ArgumentParser()
parser.add_argument("--version", required=True)
parser.add_argument("directory")
parser.add_argument("www_target")
parser.add_argument('--version', required=True)
parser.add_argument('directory')
parser.add_argument('www_target')
args = parser.parse_args()
main(args.version, args.directory, args.www_target)
tools/update-dbus-docs.py
+65 -41
View File
@@ -4,10 +4,10 @@
import argparse
import collections
import sys
import io
import os
import subprocess
import io
import sys
try:
from lxml import etree
@@ -19,9 +19,11 @@ try:
except ImportError as e:
shlex_join = e
class NoCommand(Exception):
pass
BORING_INTERFACES = [
'org.freedesktop.DBus.Peer',
'org.freedesktop.DBus.Introspectable',
@@ -34,11 +36,10 @@ RESET = '\x1b[39m'
arguments = None
def xml_parser():
return etree.XMLParser(no_network=True,
remove_comments=False,
strip_cdata=False,
resolve_entities=False)
return etree.XMLParser(no_network=True, remove_comments=False, strip_cdata=False, resolve_entities=False)
def print_method(declarations, elem, *, prefix, file, is_signal=False):
name = elem.get('name')
@@ -61,7 +62,7 @@ def print_method(declarations, elem, *, prefix, file, is_signal=False):
if argname is None:
if arguments.print_errors:
print(f'method {name}: argument {num+1} has no name', file=sys.stderr)
print(f'method {name}: argument {num + 1} has no name', file=sys.stderr)
argname = 'UNNAMED'
argtype = arg.get('type')
@@ -73,19 +74,22 @@ def print_method(declarations, elem, *, prefix, file, is_signal=False):
print(');', file=file)
ACCESS_MAP = {
'read' : 'readonly',
'write' : 'readwrite',
'read': 'readonly',
'write': 'readwrite',
}
def value_ellipsis(prop_type):
if prop_type == 's':
return "'...'"
if prop_type[0] == 'a':
inner = value_ellipsis(prop_type[1:])
return f"[{inner}{', ...' if inner != '...' else ''}]"
return f'[{inner}{", ..." if inner != "..." else ""}]'
return '...'
def print_property(declarations, elem, *, prefix, file):
prop_name = elem.get('name')
prop_type = elem.get('type')
@@ -105,11 +109,11 @@ def print_property(declarations, elem, *, prefix, file):
prop_access = ACCESS_MAP.get(prop_access, prop_access)
print(f'''{prefix}{prop_access} {prop_type} {prop_name} = {value_ellipsis(prop_type)};''', file=file)
def print_interface(iface, *, prefix, file, print_boring, only_interface, declarations):
name = iface.get('name')
is_boring = (name in BORING_INTERFACES or
only_interface is not None and name != only_interface)
is_boring = name in BORING_INTERFACES or only_interface is not None and name != only_interface
if is_boring and print_boring:
print(f'''{prefix}interface {name} {{ ... }};''', file=file)
@@ -135,10 +139,11 @@ def print_interface(iface, *, prefix, file, print_boring, only_interface, declar
print(f'''{prefix}}};''', file=file)
def check_documented(document, declarations, stats, interface, missing_version):
missing = []
sections = document.findall("refsect1")
sections = document.findall('refsect1')
history_section = document.find("refsect1[title = 'History']")
if history_section is not None:
sections.remove(history_section)
@@ -171,12 +176,13 @@ def check_documented(document, declarations, stats, interface, missing_version):
missing.append((klass, item))
if history_section is None or history_section.find(predicate) is None:
missing_version.append(f"{interface}.{item_repr}")
missing_version.append(f'{interface}.{item_repr}')
stats['missing'] += len(missing)
return missing
def xml_to_text(destination, xml, *, only_interface=None):
file = io.StringIO()
@@ -187,18 +193,23 @@ def xml_to_text(destination, xml, *, only_interface=None):
for print_boring in [False, True]:
for iface in xml.findall('./interface'):
print_interface(iface, prefix=' ', file=file,
print_boring=print_boring,
only_interface=only_interface,
declarations=declarations)
print_interface(
iface,
prefix=' ',
file=file,
print_boring=print_boring,
only_interface=only_interface,
declarations=declarations,
)
name = iface.get('name')
if not name in BORING_INTERFACES:
if name not in BORING_INTERFACES:
interfaces.append(name)
print('''};''', file=file)
return file.getvalue(), declarations, interfaces
def subst_output(document, programlisting, stats, missing_version):
executable = programlisting.get('executable', None)
if executable is None:
@@ -232,17 +243,17 @@ def subst_output(document, programlisting, stats, missing_version):
parent.remove(child)
if child.tag is etree.Comment and 'not documented' in child.text:
parent.remove(child)
if child.tag == "variablelist" and child.attrib.get("generated", False) == "True":
if child.tag == 'variablelist' and child.attrib.get('generated', False) == 'True':
parent.remove(child)
# insert pointer for systemd-directives generation
the_tail = programlisting.tail #tail is erased by addnext, so save it here.
prev_element = etree.Comment("Autogenerated cross-references for systemd.directives, do not edit")
the_tail = programlisting.tail # tail is erased by addnext, so save it here.
prev_element = etree.Comment('Autogenerated cross-references for systemd.directives, do not edit')
programlisting.addnext(prev_element)
programlisting.tail = the_tail
for interface in interfaces:
variablelist = etree.Element("variablelist")
variablelist = etree.Element('variablelist')
variablelist.attrib['class'] = 'dbus-interface'
variablelist.attrib['generated'] = 'True'
variablelist.attrib['extra-ref'] = interface
@@ -251,10 +262,10 @@ def subst_output(document, programlisting, stats, missing_version):
prev_element.tail = the_tail
prev_element = variablelist
for decl_type,decl_list in declarations.items():
for decl_type, decl_list in declarations.items():
for declaration in decl_list:
variablelist = etree.Element("variablelist")
variablelist.attrib['class'] = 'dbus-'+decl_type
variablelist = etree.Element('variablelist')
variablelist.attrib['class'] = 'dbus-' + decl_type
variablelist.attrib['generated'] = 'True'
if decl_type in ('method', 'signal'):
variablelist.attrib['extra-ref'] = declaration + '()'
@@ -265,7 +276,7 @@ def subst_output(document, programlisting, stats, missing_version):
prev_element.tail = the_tail
prev_element = variablelist
last_element = etree.Comment("End of Autogenerated section")
last_element = etree.Comment('End of Autogenerated section')
prev_element.addnext(last_element)
prev_element.tail = the_tail
last_element.tail = the_tail
@@ -276,6 +287,7 @@ def subst_output(document, programlisting, stats, missing_version):
comment.tail = programlisting.tail
parent.insert(parent.index(programlisting) + 1, comment)
def process(page, missing_version):
src = open(page).read()
xml = etree.fromstring(src, parser=xml_parser())
@@ -293,26 +305,35 @@ def process(page, missing_version):
out_text = etree.tostring(xml, encoding='unicode')
# massage format to avoid some lxml whitespace handling idiosyncrasies
# https://bugs.launchpad.net/lxml/+bug/526799
out_text = (src[:src.find('<refentryinfo')] +
out_text[out_text.find('<refentryinfo'):] +
'\n')
out_text = src[: src.find('<refentryinfo')] + out_text[out_text.find('<refentryinfo') :] + '\n'
if not arguments.test:
with open(page, 'w') as out:
out.write(out_text)
return { "stats" : stats, "modified" : out_text != src }
return {'stats': stats, 'modified': out_text != src}
def parse_args():
p = argparse.ArgumentParser()
p.add_argument('--test', action='store_true',
help='only verify that everything is up2date')
p.add_argument('--build-dir', default='build')
p.add_argument('pages', nargs='+')
p.add_argument(
'--test',
action='store_true',
help='only verify that everything is up2date',
)
p.add_argument(
'--build-dir',
default='build',
)
p.add_argument(
'pages',
nargs='+',
)
opts = p.parse_args()
opts.print_errors = not opts.test
return opts
def main():
# pylint: disable=global-statement
global arguments
@@ -327,13 +348,13 @@ def main():
sys.exit(f"{arguments.build_dir} doesn't exist.")
missing_version = []
stats = {page.split('/')[-1] : process(page, missing_version) for page in arguments.pages}
stats = {page.split('/')[-1]: process(page, missing_version) for page in arguments.pages}
ignore_list = open(os.path.join(os.path.dirname(__file__), 'dbus_ignorelist')).read().split()
missing_version = [x for x in missing_version if x not in ignore_list]
for missing in missing_version:
print(f"{RED}Missing version information for {missing}{RESET}")
print(f'{RED}Missing version information for {missing}{RESET}')
if missing_version:
sys.exit(1)
@@ -341,7 +362,7 @@ def main():
# Let's print all statistics at the end
mlen = max(len(page) for page in stats)
total = sum((item['stats'] for item in stats.values()), start=collections.Counter())
total = 'total', { "stats" : total, "modified" : False }
total = 'total', {'stats': total, 'modified': False}
modified = []
classification = 'OUTDATED' if arguments.test else 'MODIFIED'
for page, info in sorted(stats.items()) + [total]:
@@ -351,12 +372,15 @@ def main():
c = classification if info['modified'] else ''
if c:
modified.append(page)
color = RED if m > t/2 else (YELLOW if m else GREEN)
color = RED if m > t / 2 else (YELLOW if m else GREEN)
print(f'{color}{p:{mlen + 1}} {t - m}/{t} {c}{RESET}')
if arguments.test and modified:
sys.exit(f'Outdated pages: {", ".join(modified)}\n'
f'Hint: ninja -C {arguments.build_dir} update-dbus-docs')
sys.exit(
f'Outdated pages: {", ".join(modified)}\n'
f'Hint: ninja -C {arguments.build_dir} update-dbus-docs'
) # fmt: skip
if __name__ == '__main__':
main()
tools/update-man-rules.py
+15 -7
View File
@@ -13,6 +13,7 @@ from xml_helper import xml_parse
def man(page, number):
return f'{page}.{number}'
def add_rules(rules, name):
xml = xml_parse(name)
# print('parsing {}'.format(name), file=sys.stderr)
@@ -28,26 +29,28 @@ def add_rules(rules, name):
if title != refnames[0].text:
raise ValueError('refmeta and refnamediv disagree: ' + name)
for refname in refnames:
assert all(refname not in group
for group in rules.values()), "duplicate page name"
assert all(refname not in group for group in rules.values()), 'duplicate page name'
alias = man(refname.text, number)
rulegroup[alias] = target
# print('{} => {} [{}]'.format(alias, target, conditional), file=sys.stderr)
def create_rules(xml_files):
" {conditional => {alias-name => source-name}} "
"{conditional => {alias-name => source-name}}"
rules = collections.defaultdict(dict)
for name in xml_files:
try:
add_rules(rules, name)
except Exception:
print("Failed to process", name, file=sys.stderr)
print('Failed to process', name, file=sys.stderr)
raise
return rules
def mjoin(files):
return ' \\\n\t'.join(sorted(files) or '#')
MESON_HEADER = '''\
# Do not edit. Generated by update-man-rules.py.
# Update with:
@@ -59,6 +62,7 @@ MESON_FOOTER = '''\
# Really, do not edit.
'''
def make_mesonfile(rules, _dist_files):
# reformat rules as
# grouped = [ [name, section, [alias...], condition], ...]
@@ -72,10 +76,13 @@ def make_mesonfile(rules, _dist_files):
if name != alias:
group.append(alias)
lines = [ [p[0][:-2], p[0][-1], sorted(a[:-2] for a in aliases), p[1]]
for p, aliases in sorted(grouped.items()) ]
lines = [
[p[0][:-2], p[0][-1], sorted(a[:-2] for a in aliases), p[1]]
for p, aliases in sorted(grouped.items())
]
return '\n'.join((MESON_HEADER, pprint.pformat(lines)[1:-1], MESON_FOOTER))
def main():
source_glob = sys.argv[1]
target = Path(sys.argv[2])
@@ -86,7 +93,7 @@ def main():
'standard-conf.xml',
'systemd.directives.xml',
'systemd.index.xml',
'directives-template.xml'})
'directives-template.xml'}) # fmt: skip
rules = create_rules(pages)
dist_files = (Path(p).name for p in pages)
@@ -96,5 +103,6 @@ def main():
tmp.write_text(text)
tmp.rename(target)
if __name__ == '__main__':
main()
tools/xml_helper.py
+3
View File
@@ -15,14 +15,17 @@ class CustomResolver(tree.Resolver):
return None
_parser = tree.XMLParser()
# pylint: disable=no-member
_parser.resolvers.add(CustomResolver())
def xml_parse(page):
doc = tree.parse(page, _parser)
doc.xinclude()
return doc
def xml_print(xml):
return tree.tostring(xml, pretty_print=True, encoding='utf-8')