binman: Add a utility library for coreboot CBFS
Coreboot uses a simple flash-based filesystem called Coreboot Filesystem
(CBFS) to organise files used during boot. This allows files to be named
and their position in the flash to be set. It has special features for
dealing with x86 devices which typically memory-map their SPI flash to the
top of 32-bit address space and need a 'boot block' ending there.
Create a library to help create and read CBFS files. This includes a
writer class, a reader class and associated other helpers. Only a subset
of features are currently supported.
Signed-off-by: Simon Glass <sjg@chromium.org>
diff --git a/tools/binman/cbfs_util_test.py b/tools/binman/cbfs_util_test.py
new file mode 100755
index 0000000..1908630
--- /dev/null
+++ b/tools/binman/cbfs_util_test.py
@@ -0,0 +1,540 @@
+#!/usr/bin/env python
+# SPDX-License-Identifier: GPL-2.0+
+# Copyright 2019 Google LLC
+# Written by Simon Glass <sjg@chromium.org>
+
+"""Tests for cbfs_util
+
+These create and read various CBFSs and compare the results with expected
+values and with cbfstool
+"""
+
+from __future__ import print_function
+
+import io
+import os
+import shutil
+import struct
+import tempfile
+import unittest
+
+import cbfs_util
+from cbfs_util import CbfsWriter
+import elf
+import test_util
+import tools
+
+U_BOOT_DATA = b'1234'
+U_BOOT_DTB_DATA = b'udtb'
+COMPRESS_DATA = b'compress xxxxxxxxxxxxxxxxxxxxxx data'
+
+
+class TestCbfs(unittest.TestCase):
+ """Test of cbfs_util classes"""
+ #pylint: disable=W0212
+ @classmethod
+ def setUpClass(cls):
+ # Create a temporary directory for test files
+ cls._indir = tempfile.mkdtemp(prefix='cbfs_util.')
+ tools.SetInputDirs([cls._indir])
+
+ # Set up some useful data files
+ TestCbfs._make_input_file('u-boot.bin', U_BOOT_DATA)
+ TestCbfs._make_input_file('u-boot.dtb', U_BOOT_DTB_DATA)
+ TestCbfs._make_input_file('compress', COMPRESS_DATA)
+
+ # Set up a temporary output directory, used by the tools library when
+ # compressing files
+ tools.PrepareOutputDir(None)
+
+ cls.have_cbfstool = True
+ try:
+ tools.Run('which', 'cbfstool')
+ except:
+ cls.have_cbfstool = False
+
+ cls.have_lz4 = True
+ try:
+ tools.Run('lz4', '--no-frame-crc', '-c',
+ tools.GetInputFilename('u-boot.bin'))
+ except:
+ cls.have_lz4 = False
+
+ @classmethod
+ def tearDownClass(cls):
+ """Remove the temporary input directory and its contents"""
+ if cls._indir:
+ shutil.rmtree(cls._indir)
+ cls._indir = None
+ tools.FinaliseOutputDir()
+
+ @classmethod
+ def _make_input_file(cls, fname, contents):
+ """Create a new test input file, creating directories as needed
+
+ Args:
+ fname: Filename to create
+ contents: File contents to write in to the file
+ Returns:
+ Full pathname of file created
+ """
+ pathname = os.path.join(cls._indir, fname)
+ tools.WriteFile(pathname, contents)
+ return pathname
+
+ def _check_hdr(self, data, size, offset=0, arch=cbfs_util.ARCHITECTURE_X86):
+ """Check that the CBFS has the expected header
+
+ Args:
+ data: Data to check
+ size: Expected ROM size
+ offset: Expected offset to first CBFS file
+ arch: Expected architecture
+
+ Returns:
+ CbfsReader object containing the CBFS
+ """
+ cbfs = cbfs_util.CbfsReader(data)
+ self.assertEqual(cbfs_util.HEADER_MAGIC, cbfs.magic)
+ self.assertEqual(cbfs_util.HEADER_VERSION2, cbfs.version)
+ self.assertEqual(size, cbfs.rom_size)
+ self.assertEqual(0, cbfs.boot_block_size)
+ self.assertEqual(cbfs_util.ENTRY_ALIGN, cbfs.align)
+ self.assertEqual(offset, cbfs.cbfs_offset)
+ self.assertEqual(arch, cbfs.arch)
+ return cbfs
+
+ def _check_uboot(self, cbfs, ftype=cbfs_util.TYPE_RAW, offset=0x38,
+ data=U_BOOT_DATA):
+ """Check that the U-Boot file is as expected
+
+ Args:
+ cbfs: CbfsReader object to check
+ ftype: Expected file type
+ offset: Expected offset of file
+ data: Expected data in file
+
+ Returns:
+ CbfsFile object containing the file
+ """
+ self.assertIn('u-boot', cbfs.files)
+ cfile = cbfs.files['u-boot']
+ self.assertEqual('u-boot', cfile.name)
+ self.assertEqual(offset, cfile.offset)
+ self.assertEqual(data, cfile.data)
+ self.assertEqual(ftype, cfile.ftype)
+ self.assertEqual(cbfs_util.COMPRESS_NONE, cfile.compress)
+ self.assertEqual(len(data), cfile.memlen)
+ return cfile
+
+ def _check_dtb(self, cbfs, offset=0x38, data=U_BOOT_DTB_DATA):
+ """Check that the U-Boot dtb file is as expected
+
+ Args:
+ cbfs: CbfsReader object to check
+ offset: Expected offset of file
+ data: Expected data in file
+ """
+ self.assertIn('u-boot-dtb', cbfs.files)
+ cfile = cbfs.files['u-boot-dtb']
+ self.assertEqual('u-boot-dtb', cfile.name)
+ self.assertEqual(offset, cfile.offset)
+ self.assertEqual(U_BOOT_DTB_DATA, cfile.data)
+ self.assertEqual(cbfs_util.TYPE_RAW, cfile.ftype)
+ self.assertEqual(cbfs_util.COMPRESS_NONE, cfile.compress)
+ self.assertEqual(len(U_BOOT_DTB_DATA), cfile.memlen)
+
+ def _check_raw(self, data, size, offset=0, arch=cbfs_util.ARCHITECTURE_X86):
+ """Check that two raw files are added as expected
+
+ Args:
+ data: Data to check
+ size: Expected ROM size
+ offset: Expected offset to first CBFS file
+ arch: Expected architecture
+ """
+ cbfs = self._check_hdr(data, size, offset=offset, arch=arch)
+ self._check_uboot(cbfs)
+ self._check_dtb(cbfs)
+
+ def _get_expected_cbfs(self, size, arch='x86', compress=None):
+ """Get the file created by cbfstool for a particular scenario
+
+ Args:
+ size: Size of the CBFS in bytes
+ arch: Architecture of the CBFS, as a string
+ compress: Compression to use, e.g. cbfs_util.COMPRESS_LZMA
+
+ Returns:
+ Resulting CBFS file, or None if cbfstool is not available
+ """
+ if not self.have_cbfstool or not self.have_lz4:
+ return None
+ cbfs_fname = os.path.join(self._indir, 'test.cbfs')
+ cbfs_util.cbfstool(cbfs_fname, 'create', '-m', arch, '-s', '%#x' % size)
+ cbfs_util.cbfstool(cbfs_fname, 'add', '-n', 'u-boot', '-t', 'raw',
+ '-c', compress and compress[0] or 'none',
+ '-f', tools.GetInputFilename(
+ compress and 'compress' or 'u-boot.bin'))
+ cbfs_util.cbfstool(cbfs_fname, 'add', '-n', 'u-boot-dtb', '-t', 'raw',
+ '-c', compress and compress[1] or 'none',
+ '-f', tools.GetInputFilename(
+ compress and 'compress' or 'u-boot.dtb'))
+ return cbfs_fname
+
+ def _compare_expected_cbfs(self, data, cbfstool_fname):
+ """Compare against what cbfstool creates
+
+ This compares what binman creates with what cbfstool creates for what
+ is proportedly the same thing.
+
+ Args:
+ data: CBFS created by binman
+ cbfstool_fname: CBFS created by cbfstool
+ """
+ if not self.have_cbfstool or not self.have_lz4:
+ return
+ expect = tools.ReadFile(cbfstool_fname)
+ if expect != data:
+ tools.WriteFile('/tmp/expect', expect)
+ tools.WriteFile('/tmp/actual', data)
+ print('diff -y <(xxd -g1 /tmp/expect) <(xxd -g1 /tmp/actual) | colordiff')
+ self.fail('cbfstool produced a different result')
+
+ def test_cbfs_functions(self):
+ """Test global functions of cbfs_util"""
+ self.assertEqual(cbfs_util.ARCHITECTURE_X86, cbfs_util.find_arch('x86'))
+ self.assertIsNone(cbfs_util.find_arch('bad-arch'))
+
+ self.assertEqual(cbfs_util.COMPRESS_LZMA, cbfs_util.find_compress('lzma'))
+ self.assertIsNone(cbfs_util.find_compress('bad-comp'))
+
+ def test_cbfstool_failure(self):
+ """Test failure to run cbfstool"""
+ if not self.have_cbfstool:
+ self.skipTest('No cbfstool available')
+ try:
+ # In verbose mode this test fails since stderr is not captured. Fix
+ # this by turning off verbosity.
+ old_verbose = cbfs_util.VERBOSE
+ cbfs_util.VERBOSE = False
+ with test_util.capture_sys_output() as (_stdout, stderr):
+ with self.assertRaises(Exception) as e:
+ cbfs_util.cbfstool('missing-file', 'bad-command')
+ finally:
+ cbfs_util.VERBOSE = old_verbose
+ self.assertIn('Unknown command', stderr.getvalue())
+ self.assertIn('Failed to run', str(e.exception))
+
+ def test_cbfs_raw(self):
+ """Test base handling of a Coreboot Filesystem (CBFS)"""
+ size = 0xb0
+ cbw = CbfsWriter(size)
+ cbw.add_file_raw('u-boot', U_BOOT_DATA)
+ cbw.add_file_raw('u-boot-dtb', U_BOOT_DTB_DATA)
+ data = cbw.get_data()
+ self._check_raw(data, size)
+ cbfs_fname = self._get_expected_cbfs(size=size)
+ self._compare_expected_cbfs(data, cbfs_fname)
+
+ def test_cbfs_invalid_file_type(self):
+ """Check handling of an invalid file type when outputiing a CBFS"""
+ size = 0xb0
+ cbw = CbfsWriter(size)
+ cfile = cbw.add_file_raw('u-boot', U_BOOT_DATA)
+
+ # Change the type manually before generating the CBFS, and make sure
+ # that the generator complains
+ cfile.ftype = 0xff
+ with self.assertRaises(ValueError) as e:
+ cbw.get_data()
+ self.assertIn('Unknown type 0xff when writing', str(e.exception))
+
+ def test_cbfs_invalid_file_type_on_read(self):
+ """Check handling of an invalid file type when reading the CBFS"""
+ size = 0xb0
+ cbw = CbfsWriter(size)
+ cbw.add_file_raw('u-boot', U_BOOT_DATA)
+
+ data = cbw.get_data()
+
+ # Read in the first file header
+ cbr = cbfs_util.CbfsReader(data, read=False)
+ with io.BytesIO(data) as fd:
+ self.assertTrue(cbr._find_and_read_header(fd, len(data)))
+ pos = fd.tell()
+ hdr_data = fd.read(cbfs_util.FILE_HEADER_LEN)
+ magic, size, ftype, attr, offset = struct.unpack(
+ cbfs_util.FILE_HEADER_FORMAT, hdr_data)
+
+ # Create a new CBFS with a change to the file type
+ ftype = 0xff
+ newdata = data[:pos]
+ newdata += struct.pack(cbfs_util.FILE_HEADER_FORMAT, magic, size, ftype,
+ attr, offset)
+ newdata += data[pos + cbfs_util.FILE_HEADER_LEN:]
+
+ # Read in this CBFS and make sure that the reader complains
+ with self.assertRaises(ValueError) as e:
+ cbfs_util.CbfsReader(newdata)
+ self.assertIn('Unknown type 0xff when reading', str(e.exception))
+
+ def test_cbfs_no_space(self):
+ """Check handling of running out of space in the CBFS"""
+ size = 0x60
+ cbw = CbfsWriter(size)
+ cbw.add_file_raw('u-boot', U_BOOT_DATA)
+ with self.assertRaises(ValueError) as e:
+ cbw.get_data()
+ self.assertIn('No space for header', str(e.exception))
+
+ def test_cbfs_no_space_skip(self):
+ """Check handling of running out of space in CBFS with file header"""
+ size = 0x70
+ cbw = CbfsWriter(size)
+ cbw._add_fileheader = True
+ cbw.add_file_raw('u-boot', U_BOOT_DATA)
+ with self.assertRaises(ValueError) as e:
+ cbw.get_data()
+ self.assertIn('No space for data before offset', str(e.exception))
+
+ def test_cbfs_bad_header_ptr(self):
+ """Check handling of a bad master-header pointer"""
+ size = 0x70
+ cbw = CbfsWriter(size)
+ cbw.add_file_raw('u-boot', U_BOOT_DATA)
+ data = cbw.get_data()
+
+ # Add one to the pointer to make it invalid
+ newdata = data[:-4] + struct.pack('<I', cbw._header_offset + 1)
+
+ # We should still be able to find the master header by searching
+ with test_util.capture_sys_output() as (stdout, _stderr):
+ cbfs = cbfs_util.CbfsReader(newdata)
+ self.assertIn('Relative offset seems wrong', stdout.getvalue())
+ self.assertIn('u-boot', cbfs.files)
+ self.assertEqual(size, cbfs.rom_size)
+
+ def test_cbfs_bad_header(self):
+ """Check handling of a bad master header"""
+ size = 0x70
+ cbw = CbfsWriter(size)
+ cbw.add_file_raw('u-boot', U_BOOT_DATA)
+ data = cbw.get_data()
+
+ # Drop most of the header and try reading the modified CBFS
+ newdata = data[:cbw._header_offset + 4]
+
+ with test_util.capture_sys_output() as (stdout, _stderr):
+ with self.assertRaises(ValueError) as e:
+ cbfs_util.CbfsReader(newdata)
+ self.assertIn('Relative offset seems wrong', stdout.getvalue())
+ self.assertIn('Cannot find master header', str(e.exception))
+
+ def test_cbfs_bad_file_header(self):
+ """Check handling of a bad file header"""
+ size = 0x70
+ cbw = CbfsWriter(size)
+ cbw.add_file_raw('u-boot', U_BOOT_DATA)
+ data = cbw.get_data()
+
+ # Read in the CBFS master header (only), then stop
+ cbr = cbfs_util.CbfsReader(data, read=False)
+ with io.BytesIO(data) as fd:
+ self.assertTrue(cbr._find_and_read_header(fd, len(data)))
+ pos = fd.tell()
+
+ # Remove all but 4 bytes of the file headerm and try to read the file
+ newdata = data[:pos + 4]
+ with test_util.capture_sys_output() as (stdout, _stderr):
+ with io.BytesIO(newdata) as fd:
+ fd.seek(pos)
+ self.assertEqual(False, cbr._read_next_file(fd))
+ self.assertIn('File header at 0 ran out of data', stdout.getvalue())
+
+ def test_cbfs_bad_file_string(self):
+ """Check handling of an incomplete filename string"""
+ size = 0x70
+ cbw = CbfsWriter(size)
+ cbw.add_file_raw('16-characters xx', U_BOOT_DATA)
+ data = cbw.get_data()
+
+ # Read in the CBFS master header (only), then stop
+ cbr = cbfs_util.CbfsReader(data, read=False)
+ with io.BytesIO(data) as fd:
+ self.assertTrue(cbr._find_and_read_header(fd, len(data)))
+ pos = fd.tell()
+
+ # Create a new CBFS with only the first 16 bytes of the file name, then
+ # try to read the file
+ newdata = data[:pos + cbfs_util.FILE_HEADER_LEN + 16]
+ with test_util.capture_sys_output() as (stdout, _stderr):
+ with io.BytesIO(newdata) as fd:
+ fd.seek(pos)
+ self.assertEqual(False, cbr._read_next_file(fd))
+ self.assertIn('String at %x ran out of data' %
+ cbfs_util.FILE_HEADER_LEN, stdout.getvalue())
+
+ def test_cbfs_debug(self):
+ """Check debug output"""
+ size = 0x70
+ cbw = CbfsWriter(size)
+ cbw.add_file_raw('u-boot', U_BOOT_DATA)
+ data = cbw.get_data()
+
+ try:
+ cbfs_util.DEBUG = True
+ with test_util.capture_sys_output() as (stdout, _stderr):
+ cbfs_util.CbfsReader(data)
+ self.assertEqual('name u-boot\ndata %s\n' % U_BOOT_DATA,
+ stdout.getvalue())
+ finally:
+ cbfs_util.DEBUG = False
+
+ def test_cbfs_bad_attribute(self):
+ """Check handling of bad attribute tag"""
+ if not self.have_lz4:
+ self.skipTest('lz4 --no-frame-crc not available')
+ size = 0x140
+ cbw = CbfsWriter(size)
+ cbw.add_file_raw('u-boot', COMPRESS_DATA,
+ compress=cbfs_util.COMPRESS_LZ4)
+ data = cbw.get_data()
+
+ # Search the CBFS for the expected compression tag
+ with io.BytesIO(data) as fd:
+ while True:
+ pos = fd.tell()
+ tag, = struct.unpack('>I', fd.read(4))
+ if tag == cbfs_util.FILE_ATTR_TAG_COMPRESSION:
+ break
+
+ # Create a new CBFS with the tag changed to something invalid
+ newdata = data[:pos] + struct.pack('>I', 0x123) + data[pos + 4:]
+ with test_util.capture_sys_output() as (stdout, _stderr):
+ cbfs_util.CbfsReader(newdata)
+ self.assertEqual('Unknown attribute tag 123\n', stdout.getvalue())
+
+ def test_cbfs_missing_attribute(self):
+ """Check handling of an incomplete attribute tag"""
+ if not self.have_lz4:
+ self.skipTest('lz4 --no-frame-crc not available')
+ size = 0x140
+ cbw = CbfsWriter(size)
+ cbw.add_file_raw('u-boot', COMPRESS_DATA,
+ compress=cbfs_util.COMPRESS_LZ4)
+ data = cbw.get_data()
+
+ # Read in the CBFS master header (only), then stop
+ cbr = cbfs_util.CbfsReader(data, read=False)
+ with io.BytesIO(data) as fd:
+ self.assertTrue(cbr._find_and_read_header(fd, len(data)))
+ pos = fd.tell()
+
+ # Create a new CBFS with only the first 4 bytes of the compression tag,
+ # then try to read the file
+ tag_pos = pos + cbfs_util.FILE_HEADER_LEN + cbfs_util.FILENAME_ALIGN
+ newdata = data[:tag_pos + 4]
+ with test_util.capture_sys_output() as (stdout, _stderr):
+ with io.BytesIO(newdata) as fd:
+ fd.seek(pos)
+ self.assertEqual(False, cbr._read_next_file(fd))
+ self.assertIn('Attribute tag at %x ran out of data' % tag_pos,
+ stdout.getvalue())
+
+ def test_cbfs_file_master_header(self):
+ """Check handling of a file containing a master header"""
+ size = 0x100
+ cbw = CbfsWriter(size)
+ cbw._add_fileheader = True
+ cbw.add_file_raw('u-boot', U_BOOT_DATA)
+ data = cbw.get_data()
+
+ cbr = cbfs_util.CbfsReader(data)
+ self.assertIn('u-boot', cbr.files)
+ self.assertEqual(size, cbr.rom_size)
+
+ def test_cbfs_arch(self):
+ """Test on non-x86 architecture"""
+ size = 0x100
+ cbw = CbfsWriter(size, arch=cbfs_util.ARCHITECTURE_PPC64)
+ cbw.add_file_raw('u-boot', U_BOOT_DATA)
+ cbw.add_file_raw('u-boot-dtb', U_BOOT_DTB_DATA)
+ data = cbw.get_data()
+ self._check_raw(data, size, offset=0x40,
+ arch=cbfs_util.ARCHITECTURE_PPC64)
+
+ # Compare against what cbfstool creates
+ cbfs_fname = self._get_expected_cbfs(size=size, arch='ppc64')
+ self._compare_expected_cbfs(data, cbfs_fname)
+
+ def test_cbfs_stage(self):
+ """Tests handling of a Coreboot Filesystem (CBFS)"""
+ if not elf.ELF_TOOLS:
+ self.skipTest('Python elftools not available')
+ elf_fname = os.path.join(self._indir, 'cbfs-stage.elf')
+ elf.MakeElf(elf_fname, U_BOOT_DATA, U_BOOT_DTB_DATA)
+
+ size = 0xb0
+ cbw = CbfsWriter(size)
+ cbw.add_file_stage('u-boot', tools.ReadFile(elf_fname))
+
+ data = cbw.get_data()
+ cbfs = self._check_hdr(data, size)
+ load = 0xfef20000
+ entry = load + 2
+
+ cfile = self._check_uboot(cbfs, cbfs_util.TYPE_STAGE, offset=0x28,
+ data=U_BOOT_DATA + U_BOOT_DTB_DATA)
+
+ self.assertEqual(entry, cfile.entry)
+ self.assertEqual(load, cfile.load)
+ self.assertEqual(len(U_BOOT_DATA) + len(U_BOOT_DTB_DATA),
+ cfile.data_len)
+
+ # Compare against what cbfstool creates
+ if self.have_cbfstool:
+ cbfs_fname = os.path.join(self._indir, 'test.cbfs')
+ cbfs_util.cbfstool(cbfs_fname, 'create', '-m', 'x86', '-s',
+ '%#x' % size)
+ cbfs_util.cbfstool(cbfs_fname, 'add-stage', '-n', 'u-boot',
+ '-f', elf_fname)
+ self._compare_expected_cbfs(data, cbfs_fname)
+
+ def test_cbfs_raw_compress(self):
+ """Test base handling of compressing raw files"""
+ if not self.have_lz4:
+ self.skipTest('lz4 --no-frame-crc not available')
+ size = 0x140
+ cbw = CbfsWriter(size)
+ cbw.add_file_raw('u-boot', COMPRESS_DATA,
+ compress=cbfs_util.COMPRESS_LZ4)
+ cbw.add_file_raw('u-boot-dtb', COMPRESS_DATA,
+ compress=cbfs_util.COMPRESS_LZMA)
+ data = cbw.get_data()
+
+ cbfs = self._check_hdr(data, size)
+ self.assertIn('u-boot', cbfs.files)
+ cfile = cbfs.files['u-boot']
+ self.assertEqual(cfile.name, 'u-boot')
+ self.assertEqual(cfile.offset, 56)
+ self.assertEqual(cfile.data, COMPRESS_DATA)
+ self.assertEqual(cfile.ftype, cbfs_util.TYPE_RAW)
+ self.assertEqual(cfile.compress, cbfs_util.COMPRESS_LZ4)
+ self.assertEqual(cfile.memlen, len(COMPRESS_DATA))
+
+ self.assertIn('u-boot-dtb', cbfs.files)
+ cfile = cbfs.files['u-boot-dtb']
+ self.assertEqual(cfile.name, 'u-boot-dtb')
+ self.assertEqual(cfile.offset, 56)
+ self.assertEqual(cfile.data, COMPRESS_DATA)
+ self.assertEqual(cfile.ftype, cbfs_util.TYPE_RAW)
+ self.assertEqual(cfile.compress, cbfs_util.COMPRESS_LZMA)
+ self.assertEqual(cfile.memlen, len(COMPRESS_DATA))
+
+ cbfs_fname = self._get_expected_cbfs(size=size, compress=['lz4', 'lzma'])
+ self._compare_expected_cbfs(data, cbfs_fname)
+
+
+if __name__ == '__main__':
+ unittest.main()