From 2945eb73dd988cbd9775395128935256deb9a96e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Br=C3=BCns?= Date: Thu, 11 Aug 2016 22:52:03 +0200 Subject: sandbox: document support of block device emulation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stefan Brüns Acked-by: Simon Glass Changed 'Sandbox' to 'sandbox' in subject: Signed-off-by: Simon Glass diff --git a/board/sandbox/README.sandbox b/board/sandbox/README.sandbox index ed820d3..02d8ab3 100644 --- a/board/sandbox/README.sandbox +++ b/board/sandbox/README.sandbox @@ -320,6 +320,25 @@ CONFIG_SPI_IDLE_VAL The idle value on the SPI bus +Block Device Emulation +---------------------- + +U-Boot can use raw disk images for block device emulation. To e.g. list +the contents of the root directory on the second partion of the image +"disk.raw", you can use the following commands: + +=>host bind 0 ./disk.raw +=>ls host 0:2 + +A disk image can be created using the following commands: + +$> truncate -s 1200M ./disk.raw +$> echo -e "label: gpt\n,64M,U\n,,L" | /usr/sbin/sfdisk ./disk.raw +$> lodev=`sudo losetup -P -f --show ./disk.raw` +$> sudo mkfs.vfat -n EFI -v ${lodev}p1 +$> sudo mkfs.ext4 -L ROOT -v ${lodev}p2 + + Writing Sandbox Drivers ----------------------- -- cgit v0.10.2 From 49afb37988a397ce215e99563e0ab5d2970c6a39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Br=C3=BCns?= Date: Thu, 11 Aug 2016 22:52:04 +0200 Subject: sandbox: Add "host size" hostfs command for fs test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This complements the size/fatsize/ext4size commands added in commit cf6598193aed5de8855eaf70c1994f2bc437255a load, save and ls are already implemented for hostfs, now tests can cover the same operations on hostfs and emulated block devices. Signed-off-by: Stefan Brüns Acked-by: Simon Glass diff --git a/cmd/host.c b/cmd/host.c index 8d84415..b427e54 100644 --- a/cmd/host.c +++ b/cmd/host.c @@ -25,6 +25,12 @@ static int do_host_ls(cmd_tbl_t *cmdtp, int flag, int argc, return do_ls(cmdtp, flag, argc, argv, FS_TYPE_SANDBOX); } +static int do_host_size(cmd_tbl_t *cmdtp, int flag, int argc, + char * const argv[]) +{ + return do_size(cmdtp, flag, argc, argv, FS_TYPE_SANDBOX); +} + static int do_host_save(cmd_tbl_t *cmdtp, int flag, int argc, char * const argv[]) { @@ -138,6 +144,7 @@ static cmd_tbl_t cmd_host_sub[] = { U_BOOT_CMD_MKENT(load, 7, 0, do_host_load, "", ""), U_BOOT_CMD_MKENT(ls, 3, 0, do_host_ls, "", ""), U_BOOT_CMD_MKENT(save, 6, 0, do_host_save, "", ""), + U_BOOT_CMD_MKENT(size, 3, 0, do_host_size, "", ""), U_BOOT_CMD_MKENT(bind, 3, 0, do_host_bind, "", ""), U_BOOT_CMD_MKENT(info, 3, 0, do_host_info, "", ""), U_BOOT_CMD_MKENT(dev, 0, 1, do_host_dev, "", ""), @@ -174,6 +181,7 @@ U_BOOT_CMD( "host ls hostfs - - list files on host\n" "host save hostfs - [] - " "save a file to host\n" + "host size hostfs - - determine size of file on host" "host bind [] - bind \"host\" device to file\n" "host info [] - show device binding & info\n" "host dev [] - Set or retrieve the current host device\n" -- cgit v0.10.2 From 785f1548a9e74cf4796c96e6f32ed67b25f79b81 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:00 -0600 Subject: patman: Adjust command.Output() to raise an error by default It is more useful to have this method raise an error when something goes wrong. Make this the default and adjust the few callers that don't want to use it this way. Signed-off-by: Simon Glass diff --git a/tools/buildman/control.py b/tools/buildman/control.py index b86d7b3..0b6ab03 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -237,7 +237,7 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None, options.step = len(series.commits) - 1 gnu_make = command.Output(os.path.join(options.git, - 'scripts/show-gnu-make')).rstrip() + 'scripts/show-gnu-make'), raise_on_error=False).rstrip() if not gnu_make: sys.exit('GNU Make not found') diff --git a/tools/patman/checkpatch.py b/tools/patman/checkpatch.py index 34a3bd2..3eef6de 100644 --- a/tools/patman/checkpatch.py +++ b/tools/patman/checkpatch.py @@ -63,7 +63,8 @@ def CheckPatch(fname, verbose=False): result.problems = [] chk = FindCheckPatch() item = {} - result.stdout = command.Output(chk, '--no-tree', fname) + result.stdout = command.Output(chk, '--no-tree', fname, + raise_on_error=False) #pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE) #stdout, stderr = pipe.communicate() diff --git a/tools/patman/command.py b/tools/patman/command.py index d586f11..d1f0ca5 100644 --- a/tools/patman/command.py +++ b/tools/patman/command.py @@ -104,8 +104,9 @@ def RunPipe(pipe_list, infile=None, outfile=None, raise Exception("Error running '%s'" % user_pipestr) return result -def Output(*cmd): - return RunPipe([cmd], capture=True, raise_on_error=False).stdout +def Output(*cmd, **kwargs): + raise_on_error = kwargs.get('raise_on_error', True) + return RunPipe([cmd], capture=True, raise_on_error=raise_on_error).stdout def OutputOneLine(*cmd, **kwargs): raise_on_error = kwargs.pop('raise_on_error', True) diff --git a/tools/patman/gitutil.py b/tools/patman/gitutil.py index e088bae..bb7c9e0 100644 --- a/tools/patman/gitutil.py +++ b/tools/patman/gitutil.py @@ -391,7 +391,8 @@ def EmailPatches(series, cover_fname, args, dry_run, raise_on_error, cc_fname, """ to = BuildEmailList(series.get('to'), '--to', alias, raise_on_error) if not to: - git_config_to = command.Output('git', 'config', 'sendemail.to') + git_config_to = command.Output('git', 'config', 'sendemail.to', + raise_on_error=False) if not git_config_to: print ("No recipient.\n" "Please add something like this to a commit\n" -- cgit v0.10.2 From 58593115453199eb136f1c625bc29e44d057c07e Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:01 -0600 Subject: dtoc: Move the struct import into the correct order This should be in with the other system includes. Move it. Signed-off-by: Simon Glass diff --git a/tools/dtoc/dtoc.py b/tools/dtoc/dtoc.py index ec80abe..e9ab46f 100755 --- a/tools/dtoc/dtoc.py +++ b/tools/dtoc/dtoc.py @@ -9,6 +9,7 @@ import copy from optparse import OptionError, OptionParser import os +import struct import sys import fdt_util @@ -29,8 +30,6 @@ except ImportError: from fdt_fallback import Fdt import fdt_fallback as fdt -import struct - # When we see these properties we ignore them - i.e. do not create a structure member PROP_IGNORE_LIST = [ '#address-cells', -- cgit v0.10.2 From ba482585661431dde74eb69c7a94ac2d5a772d9f Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:02 -0600 Subject: dtoc: Move the fdt library selection into fdt_select Rather than have dtc worry about which fdt library to use, move this into a helper file. Add a function which creates a new Fdt object and scans it, regardless of the implementation. Signed-off-by: Simon Glass diff --git a/tools/dtoc/dtoc.py b/tools/dtoc/dtoc.py index e9ab46f..10f6a12 100755 --- a/tools/dtoc/dtoc.py +++ b/tools/dtoc/dtoc.py @@ -12,23 +12,12 @@ import os import struct import sys -import fdt_util - # Bring in the patman libraries our_path = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.join(our_path, '../patman')) -# Bring in either the normal fdt library (which relies on libfdt) or the -# fallback one (which uses fdtget and is slower). Both provide the same -# interfface for this file to use. -try: - from fdt import Fdt - import fdt - have_libfdt = True -except ImportError: - have_libfdt = False - from fdt_fallback import Fdt - import fdt_fallback as fdt +import fdt_select +import fdt_util # When we see these properties we ignore them - i.e. do not create a structure member PROP_IGNORE_LIST = [ @@ -177,8 +166,7 @@ class DtbPlatdata: Once this is done, self.fdt.GetRoot() can be called to obtain the device tree root node, and progress from there. """ - self.fdt = Fdt(self._dtb_fname) - self.fdt.Scan() + self.fdt = fdt_select.FdtScan(self._dtb_fname) def ScanTree(self): """Scan the device tree for useful information diff --git a/tools/dtoc/fdt_select.py b/tools/dtoc/fdt_select.py new file mode 100644 index 0000000..5aff297 --- /dev/null +++ b/tools/dtoc/fdt_select.py @@ -0,0 +1,23 @@ +#!/usr/bin/python +# +# Copyright (C) 2016 Google, Inc +# Written by Simon Glass +# +# SPDX-License-Identifier: GPL-2.0+ +# + +# Bring in either the normal fdt library (which relies on libfdt) or the +# fallback one (which uses fdtget and is slower). Both provide the same +# interface for this file to use. +try: + import fdt + have_libfdt = True +except ImportError: + have_libfdt = False + import fdt_fallback as fdt + +def FdtScan(fname): + """Returns a new Fdt object from the implementation we are using""" + dtb = fdt.Fdt(fname) + dtb.Scan() + return dtb -- cgit v0.10.2 From 66051b1f59dfba48566dc3e0eee4c093e1104590 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:03 -0600 Subject: dtoc: Rename fdt.py to fdt_normal.py In preparation for creating an Fdt base class, rename this file to indicate it is the normal Fdt implementation. Signed-off-by: Simon Glass diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py deleted file mode 100644 index 1d913a9..0000000 --- a/tools/dtoc/fdt.py +++ /dev/null @@ -1,180 +0,0 @@ -#!/usr/bin/python -# -# Copyright (C) 2016 Google, Inc -# Written by Simon Glass -# -# SPDX-License-Identifier: GPL-2.0+ -# - -import fdt_util -import libfdt -import sys - -# This deals with a device tree, presenting it as a list of Node and Prop -# objects, representing nodes and properties, respectively. -# -# This implementation uses a libfdt Python library to access the device tree, -# so it is fairly efficient. - -class Prop: - """A device tree property - - Properties: - name: Property name (as per the device tree) - value: Property value as a string of bytes, or a list of strings of - bytes - type: Value type - """ - def __init__(self, name, bytes): - self.name = name - self.value = None - if not bytes: - self.type = fdt_util.TYPE_BOOL - self.value = True - return - self.type, self.value = fdt_util.BytesToValue(bytes) - - def GetPhandle(self): - """Get a (single) phandle value from a property - - Gets the phandle valuie from a property and returns it as an integer - """ - return fdt_util.fdt32_to_cpu(self.value[:4]) - - def Widen(self, newprop): - """Figure out which property type is more general - - Given a current property and a new property, this function returns the - one that is less specific as to type. The less specific property will - be ble to represent the data in the more specific property. This is - used for things like: - - node1 { - compatible = "fred"; - value = <1>; - }; - node1 { - compatible = "fred"; - value = <1 2>; - }; - - He we want to use an int array for 'value'. The first property - suggests that a single int is enough, but the second one shows that - it is not. Calling this function with these two propertes would - update the current property to be like the second, since it is less - specific. - """ - if newprop.type < self.type: - self.type = newprop.type - - if type(newprop.value) == list and type(self.value) != list: - self.value = [self.value] - - if type(self.value) == list and len(newprop.value) > len(self.value): - val = fdt_util.GetEmpty(self.type) - while len(self.value) < len(newprop.value): - self.value.append(val) - - -class Node: - """A device tree node - - Properties: - offset: Integer offset in the device tree - name: Device tree node tname - path: Full path to node, along with the node name itself - _fdt: Device tree object - subnodes: A list of subnodes for this node, each a Node object - props: A dict of properties for this node, each a Prop object. - Keyed by property name - """ - def __init__(self, fdt, offset, name, path): - self.offset = offset - self.name = name - self.path = path - self._fdt = fdt - self.subnodes = [] - self.props = {} - - def Scan(self): - """Scan a node's properties and subnodes - - This fills in the props and subnodes properties, recursively - searching into subnodes so that the entire tree is built. - """ - self.props = self._fdt.GetProps(self.path) - - offset = libfdt.fdt_first_subnode(self._fdt.GetFdt(), self.offset) - while offset >= 0: - sep = '' if self.path[-1] == '/' else '/' - name = libfdt.Name(self._fdt.GetFdt(), offset) - path = self.path + sep + name - node = Node(self._fdt, offset, name, path) - self.subnodes.append(node) - - node.Scan() - offset = libfdt.fdt_next_subnode(self._fdt.GetFdt(), offset) - - -class Fdt: - """Provides simple access to a flat device tree blob. - - Properties: - fname: Filename of fdt - _root: Root of device tree (a Node object) - """ - - def __init__(self, fname): - self.fname = fname - with open(fname) as fd: - self._fdt = fd.read() - - def GetFdt(self): - """Get the contents of the FDT - - Returns: - The FDT contents as a string of bytes - """ - return self._fdt - - def Scan(self): - """Scan a device tree, building up a tree of Node objects - - This fills in the self._root property - """ - self._root = Node(self, 0, '/', '/') - self._root.Scan() - - def GetRoot(self): - """Get the root Node of the device tree - - Returns: - The root Node object - """ - return self._root - - def GetProps(self, node): - """Get all properties from a node. - - Args: - node: Full path to node name to look in. - - Returns: - A dictionary containing all the properties, indexed by node name. - The entries are Prop objects. - - Raises: - ValueError: if the node does not exist. - """ - offset = libfdt.fdt_path_offset(self._fdt, node) - if offset < 0: - libfdt.Raise(offset) - props_dict = {} - poffset = libfdt.fdt_first_property_offset(self._fdt, offset) - while poffset >= 0: - dprop, plen = libfdt.fdt_get_property_by_offset(self._fdt, poffset) - prop = Prop(libfdt.String(self._fdt, dprop.nameoff), libfdt.Data(dprop)) - props_dict[prop.name] = prop - - poffset = libfdt.fdt_next_property_offset(self._fdt, poffset) - return props_dict diff --git a/tools/dtoc/fdt_normal.py b/tools/dtoc/fdt_normal.py new file mode 100644 index 0000000..1d913a9 --- /dev/null +++ b/tools/dtoc/fdt_normal.py @@ -0,0 +1,180 @@ +#!/usr/bin/python +# +# Copyright (C) 2016 Google, Inc +# Written by Simon Glass +# +# SPDX-License-Identifier: GPL-2.0+ +# + +import fdt_util +import libfdt +import sys + +# This deals with a device tree, presenting it as a list of Node and Prop +# objects, representing nodes and properties, respectively. +# +# This implementation uses a libfdt Python library to access the device tree, +# so it is fairly efficient. + +class Prop: + """A device tree property + + Properties: + name: Property name (as per the device tree) + value: Property value as a string of bytes, or a list of strings of + bytes + type: Value type + """ + def __init__(self, name, bytes): + self.name = name + self.value = None + if not bytes: + self.type = fdt_util.TYPE_BOOL + self.value = True + return + self.type, self.value = fdt_util.BytesToValue(bytes) + + def GetPhandle(self): + """Get a (single) phandle value from a property + + Gets the phandle valuie from a property and returns it as an integer + """ + return fdt_util.fdt32_to_cpu(self.value[:4]) + + def Widen(self, newprop): + """Figure out which property type is more general + + Given a current property and a new property, this function returns the + one that is less specific as to type. The less specific property will + be ble to represent the data in the more specific property. This is + used for things like: + + node1 { + compatible = "fred"; + value = <1>; + }; + node1 { + compatible = "fred"; + value = <1 2>; + }; + + He we want to use an int array for 'value'. The first property + suggests that a single int is enough, but the second one shows that + it is not. Calling this function with these two propertes would + update the current property to be like the second, since it is less + specific. + """ + if newprop.type < self.type: + self.type = newprop.type + + if type(newprop.value) == list and type(self.value) != list: + self.value = [self.value] + + if type(self.value) == list and len(newprop.value) > len(self.value): + val = fdt_util.GetEmpty(self.type) + while len(self.value) < len(newprop.value): + self.value.append(val) + + +class Node: + """A device tree node + + Properties: + offset: Integer offset in the device tree + name: Device tree node tname + path: Full path to node, along with the node name itself + _fdt: Device tree object + subnodes: A list of subnodes for this node, each a Node object + props: A dict of properties for this node, each a Prop object. + Keyed by property name + """ + def __init__(self, fdt, offset, name, path): + self.offset = offset + self.name = name + self.path = path + self._fdt = fdt + self.subnodes = [] + self.props = {} + + def Scan(self): + """Scan a node's properties and subnodes + + This fills in the props and subnodes properties, recursively + searching into subnodes so that the entire tree is built. + """ + self.props = self._fdt.GetProps(self.path) + + offset = libfdt.fdt_first_subnode(self._fdt.GetFdt(), self.offset) + while offset >= 0: + sep = '' if self.path[-1] == '/' else '/' + name = libfdt.Name(self._fdt.GetFdt(), offset) + path = self.path + sep + name + node = Node(self._fdt, offset, name, path) + self.subnodes.append(node) + + node.Scan() + offset = libfdt.fdt_next_subnode(self._fdt.GetFdt(), offset) + + +class Fdt: + """Provides simple access to a flat device tree blob. + + Properties: + fname: Filename of fdt + _root: Root of device tree (a Node object) + """ + + def __init__(self, fname): + self.fname = fname + with open(fname) as fd: + self._fdt = fd.read() + + def GetFdt(self): + """Get the contents of the FDT + + Returns: + The FDT contents as a string of bytes + """ + return self._fdt + + def Scan(self): + """Scan a device tree, building up a tree of Node objects + + This fills in the self._root property + """ + self._root = Node(self, 0, '/', '/') + self._root.Scan() + + def GetRoot(self): + """Get the root Node of the device tree + + Returns: + The root Node object + """ + return self._root + + def GetProps(self, node): + """Get all properties from a node. + + Args: + node: Full path to node name to look in. + + Returns: + A dictionary containing all the properties, indexed by node name. + The entries are Prop objects. + + Raises: + ValueError: if the node does not exist. + """ + offset = libfdt.fdt_path_offset(self._fdt, node) + if offset < 0: + libfdt.Raise(offset) + props_dict = {} + poffset = libfdt.fdt_first_property_offset(self._fdt, offset) + while poffset >= 0: + dprop, plen = libfdt.fdt_get_property_by_offset(self._fdt, poffset) + prop = Prop(libfdt.String(self._fdt, dprop.nameoff), libfdt.Data(dprop)) + props_dict[prop.name] = prop + + poffset = libfdt.fdt_next_property_offset(self._fdt, poffset) + return props_dict diff --git a/tools/dtoc/fdt_select.py b/tools/dtoc/fdt_select.py index 5aff297..681dfbf 100644 --- a/tools/dtoc/fdt_select.py +++ b/tools/dtoc/fdt_select.py @@ -10,7 +10,7 @@ # fallback one (which uses fdtget and is slower). Both provide the same # interface for this file to use. try: - import fdt + import fdt_normal as fdt have_libfdt = True except ImportError: have_libfdt = False -- cgit v0.10.2 From a06a34b2031e0797892e188595bfb305cd9719ab Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:04 -0600 Subject: dtoc: Create a base class for Fdt At present we have two separate implementations of the Fdt library, one which uses fdtget/fdtput and one which uses libfdt (via swig). Before adding more functionality it makes sense to create a base class for these. This will allow common functions to be shared, and make the Fdt API a little clearer. Create a new fdt.py file with the base class, and adjust fdt_normal.py and fdt_fallback.py to use it. Signed-off-by: Simon Glass diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py new file mode 100644 index 0000000..413d45d --- /dev/null +++ b/tools/dtoc/fdt.py @@ -0,0 +1,68 @@ +#!/usr/bin/python +# +# Copyright (C) 2016 Google, Inc +# Written by Simon Glass +# +# SPDX-License-Identifier: GPL-2.0+ +# + +import struct +import sys + +import fdt_util + +# This deals with a device tree, presenting it as an assortment of Node and +# Prop objects, representing nodes and properties, respectively. This file +# contains the base classes and defines the high-level API. Most of the +# implementation is in the FdtFallback and FdtNormal subclasses. See +# fdt_select.py for how to create an Fdt object. + +def CheckErr(errnum, msg): + if errnum: + raise ValueError('Error %d: %s: %s' % + (errnum, libfdt.fdt_strerror(errnum), msg)) + +class PropBase: + """A device tree property + + Properties: + name: Property name (as per the device tree) + value: Property value as a string of bytes, or a list of strings of + bytes + type: Value type + """ + def __init__(self, node, offset, name): + self._node = node + self._offset = offset + self.name = name + self.value = None + +class NodeBase: + """A device tree node + + Properties: + offset: Integer offset in the device tree + name: Device tree node tname + path: Full path to node, along with the node name itself + _fdt: Device tree object + subnodes: A list of subnodes for this node, each a Node object + props: A dict of properties for this node, each a Prop object. + Keyed by property name + """ + def __init__(self, fdt, offset, name, path): + self._fdt = fdt + self._offset = offset + self.name = name + self.path = path + self.subnodes = [] + self.props = {} + +class Fdt: + """Provides simple access to a flat device tree blob. + + Properties: + fname: Filename of fdt + _root: Root of device tree (a Node object) + """ + def __init__(self, fname): + self._fname = fname diff --git a/tools/dtoc/fdt_fallback.py b/tools/dtoc/fdt_fallback.py index 9ed11e4..be3b5ba 100644 --- a/tools/dtoc/fdt_fallback.py +++ b/tools/dtoc/fdt_fallback.py @@ -7,6 +7,8 @@ # import command +import fdt +from fdt import Fdt, NodeBase, PropBase import fdt_util import sys @@ -17,7 +19,7 @@ import sys # is not very efficient for larger trees. The tool is called once for each # node and property in the tree. -class Prop: +class Prop(PropBase): """A device tree property Properties: @@ -26,14 +28,14 @@ class Prop: bytes type: Value type """ - def __init__(self, name, byte_list_str): - self.name = name - self.value = None + def __init__(self, node, name, byte_list_str): + PropBase.__init__(self, node, 0, name) if not byte_list_str.strip(): self.type = fdt_util.TYPE_BOOL return - bytes = [chr(int(byte, 16)) for byte in byte_list_str.strip().split(' ')] - self.type, self.value = fdt_util.BytesToValue(''.join(bytes)) + self.bytes = [chr(int(byte, 16)) + for byte in byte_list_str.strip().split(' ')] + self.type, self.value = fdt_util.BytesToValue(''.join(self.bytes)) def GetPhandle(self): """Get a (single) phandle value from a property @@ -77,7 +79,7 @@ class Prop: self.value.append(val) -class Node: +class Node(NodeBase): """A device tree node Properties: @@ -88,12 +90,8 @@ class Node: props: A dict of properties for this node, each a Prop object. Keyed by property name """ - def __init__(self, fdt, name, path): - self.name = name - self.path = path - self._fdt = fdt - self.subnodes = [] - self.props = {} + def __init__(self, fdt, offset, name, path): + NodeBase.__init__(self, fdt, offset, name, path) def Scan(self): """Scan a node's properties and subnodes @@ -102,35 +100,34 @@ class Node: searching into subnodes so that the entire tree is built. """ for name, byte_list_str in self._fdt.GetProps(self.path).iteritems(): - prop = Prop(name, byte_list_str) + prop = Prop(self, name, byte_list_str) self.props[name] = prop for name in self._fdt.GetSubNodes(self.path): sep = '' if self.path[-1] == '/' else '/' path = self.path + sep + name - node = Node(self._fdt, name, path) + node = Node(self._fdt, 0, name, path) self.subnodes.append(node) node.Scan() -class Fdt: - """Provides simple access to a flat device tree blob. +class FdtFallback(Fdt): + """Provides simple access to a flat device tree blob using fdtget/fdtput Properties: - fname: Filename of fdt - _root: Root of device tree (a Node object) + See superclass """ def __init__(self, fname): - self.fname = fname + Fdt.__init__(self, fname) def Scan(self): """Scan a device tree, building up a tree of Node objects This fills in the self._root property """ - self._root = Node(self, '/', '/') + self._root = Node(self, 0, '/', '/') self._root.Scan() def GetRoot(self): @@ -153,7 +150,7 @@ class Fdt: Raises: CmdError: if the node does not exist. """ - out = command.Output('fdtget', self.fname, '-l', node) + out = command.Output('fdtget', self._fname, '-l', node) return out.strip().splitlines() def GetProps(self, node, convert_dashes=False): @@ -171,7 +168,7 @@ class Fdt: Raises: CmdError: if the node does not exist. """ - out = command.Output('fdtget', self.fname, node, '-p') + out = command.Output('fdtget', self._fname, node, '-p') props = out.strip().splitlines() props_dict = {} for prop in props: @@ -204,10 +201,26 @@ class Fdt: Raises: CmdError: if the property does not exist and no default is provided. """ - args = [self.fname, node, prop, '-t', 'bx'] + args = [self._fname, node, prop, '-t', 'bx'] if default is not None: args += ['-d', str(default)] if typespec is not None: args += ['-t%s' % typespec] out = command.Output('fdtget', *args) return out.strip() + + @classmethod + def Node(self, fdt, offset, name, path): + """Create a new node + + This is used by Fdt.Scan() to create a new node using the correct + class. + + Args: + fdt: Fdt object + offset: Offset of node + name: Node name + path: Full path to node + """ + node = Node(fdt, offset, name, path) + return node diff --git a/tools/dtoc/fdt_normal.py b/tools/dtoc/fdt_normal.py index 1d913a9..ca5335b 100644 --- a/tools/dtoc/fdt_normal.py +++ b/tools/dtoc/fdt_normal.py @@ -6,9 +6,13 @@ # SPDX-License-Identifier: GPL-2.0+ # +import struct +import sys + +import fdt +from fdt import Fdt, NodeBase, PropBase import fdt_util import libfdt -import sys # This deals with a device tree, presenting it as a list of Node and Prop # objects, representing nodes and properties, respectively. @@ -16,7 +20,7 @@ import sys # This implementation uses a libfdt Python library to access the device tree, # so it is fairly efficient. -class Prop: +class Prop(PropBase): """A device tree property Properties: @@ -25,9 +29,9 @@ class Prop: bytes type: Value type """ - def __init__(self, name, bytes): - self.name = name - self.value = None + def __init__(self, node, offset, name, bytes): + PropBase.__init__(self, node, offset, name) + self.bytes = bytes if not bytes: self.type = fdt_util.TYPE_BOOL self.value = True @@ -76,7 +80,7 @@ class Prop: self.value.append(val) -class Node: +class Node(NodeBase): """A device tree node Properties: @@ -89,12 +93,7 @@ class Node: Keyed by property name """ def __init__(self, fdt, offset, name, path): - self.offset = offset - self.name = name - self.path = path - self._fdt = fdt - self.subnodes = [] - self.props = {} + NodeBase.__init__(self, fdt, offset, name, path) def Scan(self): """Scan a node's properties and subnodes @@ -104,7 +103,7 @@ class Node: """ self.props = self._fdt.GetProps(self.path) - offset = libfdt.fdt_first_subnode(self._fdt.GetFdt(), self.offset) + offset = libfdt.fdt_first_subnode(self._fdt.GetFdt(), self._offset) while offset >= 0: sep = '' if self.path[-1] == '/' else '/' name = libfdt.Name(self._fdt.GetFdt(), offset) @@ -116,17 +115,17 @@ class Node: offset = libfdt.fdt_next_subnode(self._fdt.GetFdt(), offset) -class Fdt: - """Provides simple access to a flat device tree blob. +class FdtNormal(Fdt): + """Provides simple access to a flat device tree blob using libfdt. Properties: - fname: Filename of fdt - _root: Root of device tree (a Node object) + _fdt: Device tree contents (bytearray) + _cached_offsets: True if all the nodes have a valid _offset property, + False if something has changed to invalidate the offsets """ - def __init__(self, fname): - self.fname = fname - with open(fname) as fd: + Fdt.__init__(self, fname) + with open(self._fname) as fd: self._fdt = fd.read() def GetFdt(self): @@ -173,8 +172,25 @@ class Fdt: poffset = libfdt.fdt_first_property_offset(self._fdt, offset) while poffset >= 0: dprop, plen = libfdt.fdt_get_property_by_offset(self._fdt, poffset) - prop = Prop(libfdt.String(self._fdt, dprop.nameoff), libfdt.Data(dprop)) + prop = Prop(node, poffset, libfdt.String(self._fdt, dprop.nameoff), + libfdt.Data(dprop)) props_dict[prop.name] = prop poffset = libfdt.fdt_next_property_offset(self._fdt, poffset) return props_dict + + @classmethod + def Node(self, fdt, offset, name, path): + """Create a new node + + This is used by Fdt.Scan() to create a new node using the correct + class. + + Args: + fdt: Fdt object + offset: Offset of node + name: Node name + path: Full path to node + """ + node = Node(fdt, offset, name, path) + return node diff --git a/tools/dtoc/fdt_select.py b/tools/dtoc/fdt_select.py index 681dfbf..18a36d8 100644 --- a/tools/dtoc/fdt_select.py +++ b/tools/dtoc/fdt_select.py @@ -10,14 +10,17 @@ # fallback one (which uses fdtget and is slower). Both provide the same # interface for this file to use. try: - import fdt_normal as fdt + import fdt_normal have_libfdt = True except ImportError: have_libfdt = False - import fdt_fallback as fdt + import fdt_fallback def FdtScan(fname): """Returns a new Fdt object from the implementation we are using""" - dtb = fdt.Fdt(fname) + if have_libfdt: + dtb = fdt_normal.FdtNormal(fname) + else: + dtb = fdt_fallback.FdtFallback(fname) dtb.Scan() return dtb -- cgit v0.10.2 From bc1dea3656e55d91f7a3c1339d53fc9def3bbf31 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:05 -0600 Subject: dtoc: Move BytesToValue() and GetEmpty() into PropBase These functions are currently in a separate fdt_util file. Since they are only used from PropBase and subclasses, it makes sense for them to be in the PropBase class. Move these functions into fdt.py along with the list of types. Signed-off-by: Simon Glass diff --git a/tools/dtoc/dtoc.py b/tools/dtoc/dtoc.py index 10f6a12..518aa51 100755 --- a/tools/dtoc/dtoc.py +++ b/tools/dtoc/dtoc.py @@ -16,6 +16,7 @@ import sys our_path = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.join(our_path, '../patman')) +import fdt import fdt_select import fdt_util @@ -33,10 +34,10 @@ PROP_IGNORE_LIST = [ # C type declarations for the tyues we support TYPE_NAMES = { - fdt_util.TYPE_INT: 'fdt32_t', - fdt_util.TYPE_BYTE: 'unsigned char', - fdt_util.TYPE_STRING: 'const char *', - fdt_util.TYPE_BOOL: 'bool', + fdt.TYPE_INT: 'fdt32_t', + fdt.TYPE_BYTE: 'unsigned char', + fdt.TYPE_STRING: 'const char *', + fdt.TYPE_BOOL: 'bool', }; STRUCT_PREFIX = 'dtd_' @@ -138,13 +139,13 @@ class DtbPlatdata: type: Data type (fdt_util) value: Data value, as a string of bytes """ - if type == fdt_util.TYPE_INT: + if type == fdt.TYPE_INT: return '%#x' % fdt_util.fdt32_to_cpu(value) - elif type == fdt_util.TYPE_BYTE: + elif type == fdt.TYPE_BYTE: return '%#x' % ord(value[0]) - elif type == fdt_util.TYPE_STRING: + elif type == fdt.TYPE_STRING: return '"%s"' % value - elif type == fdt_util.TYPE_BOOL: + elif type == fdt.TYPE_BOOL: return 'true' def GetCompatName(self, node): diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py index 413d45d..329d03c 100644 --- a/tools/dtoc/fdt.py +++ b/tools/dtoc/fdt.py @@ -17,6 +17,9 @@ import fdt_util # implementation is in the FdtFallback and FdtNormal subclasses. See # fdt_select.py for how to create an Fdt object. +# A list of types we support +(TYPE_BYTE, TYPE_INT, TYPE_STRING, TYPE_BOOL) = range(4) + def CheckErr(errnum, msg): if errnum: raise ValueError('Error %d: %s: %s' % @@ -37,6 +40,69 @@ class PropBase: self.name = name self.value = None + def BytesToValue(self, bytes): + """Converts a string of bytes into a type and value + + Args: + A string containing bytes + + Return: + A tuple: + Type of data + Data, either a single element or a list of elements. Each element + is one of: + TYPE_STRING: string value from the property + TYPE_INT: a byte-swapped integer stored as a 4-byte string + TYPE_BYTE: a byte stored as a single-byte string + """ + size = len(bytes) + strings = bytes.split('\0') + is_string = True + count = len(strings) - 1 + if count > 0 and not strings[-1]: + for string in strings[:-1]: + if not string: + is_string = False + break + for ch in string: + if ch < ' ' or ch > '~': + is_string = False + break + else: + is_string = False + if is_string: + if count == 1: + return TYPE_STRING, strings[0] + else: + return TYPE_STRING, strings[:-1] + if size % 4: + if size == 1: + return TYPE_BYTE, bytes[0] + else: + return TYPE_BYTE, list(bytes) + val = [] + for i in range(0, size, 4): + val.append(bytes[i:i + 4]) + if size == 4: + return TYPE_INT, val[0] + else: + return TYPE_INT, val + + def GetEmpty(self, type): + """Get an empty / zero value of the given type + + Returns: + A single value of the given type + """ + if type == TYPE_BYTE: + return chr(0) + elif type == TYPE_INT: + return struct.pack(' len(self.value): - val = fdt_util.GetEmpty(self.type) + val = self.GetEmpty(self.type) while len(self.value) < len(newprop.value): self.value.append(val) diff --git a/tools/dtoc/fdt_util.py b/tools/dtoc/fdt_util.py index 929b524..6b57248 100644 --- a/tools/dtoc/fdt_util.py +++ b/tools/dtoc/fdt_util.py @@ -8,72 +8,6 @@ import struct -# A list of types we support -(TYPE_BYTE, TYPE_INT, TYPE_STRING, TYPE_BOOL) = range(4) - -def BytesToValue(bytes): - """Converts a string of bytes into a type and value - - Args: - A string containing bytes - - Return: - A tuple: - Type of data - Data, either a single element or a list of elements. Each element - is one of: - TYPE_STRING: string value from the property - TYPE_INT: a byte-swapped integer stored as a 4-byte string - TYPE_BYTE: a byte stored as a single-byte string - """ - size = len(bytes) - strings = bytes.split('\0') - is_string = True - count = len(strings) - 1 - if count > 0 and not strings[-1]: - for string in strings[:-1]: - if not string: - is_string = False - break - for ch in string: - if ch < ' ' or ch > '~': - is_string = False - break - else: - is_string = False - if is_string: - if count == 1: - return TYPE_STRING, strings[0] - else: - return TYPE_STRING, strings[:-1] - if size % 4: - if size == 1: - return TYPE_BYTE, bytes[0] - else: - return TYPE_BYTE, list(bytes) - val = [] - for i in range(0, size, 4): - val.append(bytes[i:i + 4]) - if size == 4: - return TYPE_INT, val[0] - else: - return TYPE_INT, val - -def GetEmpty(type): - """Get an empty / zero value of the given type - - Returns: - A single value of the given type - """ - if type == TYPE_BYTE: - return chr(0) - elif type == TYPE_INT: - return struct.pack(' Date: Mon, 25 Jul 2016 18:59:06 -0600 Subject: dtoc: Move Widen() and GetPhandle() into the base class These functions are identical in both subclasses. Move them into the base class. Note: In fact there is a bug in one version, which was fixed by this patch: https://patchwork.ozlabs.org/patch/651697/ Signed-off-by: Simon Glass diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py index 329d03c..964ef7c 100644 --- a/tools/dtoc/fdt.py +++ b/tools/dtoc/fdt.py @@ -40,6 +40,47 @@ class PropBase: self.name = name self.value = None + def GetPhandle(self): + """Get a (single) phandle value from a property + + Gets the phandle valuie from a property and returns it as an integer + """ + return fdt_util.fdt32_to_cpu(self.value[:4]) + + def Widen(self, newprop): + """Figure out which property type is more general + + Given a current property and a new property, this function returns the + one that is less specific as to type. The less specific property will + be ble to represent the data in the more specific property. This is + used for things like: + + node1 { + compatible = "fred"; + value = <1>; + }; + node1 { + compatible = "fred"; + value = <1 2>; + }; + + He we want to use an int array for 'value'. The first property + suggests that a single int is enough, but the second one shows that + it is not. Calling this function with these two propertes would + update the current property to be like the second, since it is less + specific. + """ + if newprop.type < self.type: + self.type = newprop.type + + if type(newprop.value) == list and type(self.value) != list: + self.value = [self.value] + + if type(self.value) == list and len(newprop.value) > len(self.value): + val = self.GetEmpty(self.type) + while len(self.value) < len(newprop.value): + self.value.append(val) + def BytesToValue(self, bytes): """Converts a string of bytes into a type and value diff --git a/tools/dtoc/fdt_fallback.py b/tools/dtoc/fdt_fallback.py index 798c510..84a3db1 100644 --- a/tools/dtoc/fdt_fallback.py +++ b/tools/dtoc/fdt_fallback.py @@ -37,47 +37,6 @@ class Prop(PropBase): for byte in byte_list_str.strip().split(' ')] self.type, self.value = self.BytesToValue(''.join(self.bytes)) - def GetPhandle(self): - """Get a (single) phandle value from a property - - Gets the phandle valuie from a property and returns it as an integer - """ - return fdt_util.fdt32_to_cpu(self.value[:4]) - - def Widen(self, newprop): - """Figure out which property type is more general - - Given a current property and a new property, this function returns the - one that is less specific as to type. The less specific property will - be ble to represent the data in the more specific property. This is - used for things like: - - node1 { - compatible = "fred"; - value = <1>; - }; - node1 { - compatible = "fred"; - value = <1 2>; - }; - - He we want to use an int array for 'value'. The first property - suggests that a single int is enough, but the second one shows that - it is not. Calling this function with these two propertes would - update the current property to be like the second, since it is less - specific. - """ - if newprop.type < self.type: - self.type = newprop.type - - if type(newprop.value) == list and type(self.value) != list: - self.value = newprop.value - - if type(self.value) == list and len(newprop.value) > len(self.value): - val = fdt_util.GetEmpty(self.type) - while len(self.value) < len(newprop.value): - self.value.append(val) - class Node(NodeBase): """A device tree node diff --git a/tools/dtoc/fdt_normal.py b/tools/dtoc/fdt_normal.py index c7c86b8..6f019c1 100644 --- a/tools/dtoc/fdt_normal.py +++ b/tools/dtoc/fdt_normal.py @@ -38,48 +38,6 @@ class Prop(PropBase): return self.type, self.value = self.BytesToValue(bytes) - def GetPhandle(self): - """Get a (single) phandle value from a property - - Gets the phandle valuie from a property and returns it as an integer - """ - return fdt_util.fdt32_to_cpu(self.value[:4]) - - def Widen(self, newprop): - """Figure out which property type is more general - - Given a current property and a new property, this function returns the - one that is less specific as to type. The less specific property will - be ble to represent the data in the more specific property. This is - used for things like: - - node1 { - compatible = "fred"; - value = <1>; - }; - node1 { - compatible = "fred"; - value = <1 2>; - }; - - He we want to use an int array for 'value'. The first property - suggests that a single int is enough, but the second one shows that - it is not. Calling this function with these two propertes would - update the current property to be like the second, since it is less - specific. - """ - if newprop.type < self.type: - self.type = newprop.type - - if type(newprop.value) == list and type(self.value) != list: - self.value = [self.value] - - if type(self.value) == list and len(newprop.value) > len(self.value): - val = self.GetEmpty(self.type) - while len(self.value) < len(newprop.value): - self.value.append(val) - - class Node(NodeBase): """A device tree node -- cgit v0.10.2 From f7a2aeeeb8d4bbbb9bfd788903942062c8535efb Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:07 -0600 Subject: dtoc: Move a few more common functions into fdt.py Some functions have the same code in the subclasses. Move these into the superclass to avoid duplication. Signed-off-by: Simon Glass diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py index 964ef7c..c0ce5af 100644 --- a/tools/dtoc/fdt.py +++ b/tools/dtoc/fdt.py @@ -164,6 +164,26 @@ class NodeBase: self.subnodes = [] self.props = {} + def _FindNode(self, name): + """Find a node given its name + + Args: + name: Node name to look for + Returns: + Node object if found, else None + """ + for subnode in self.subnodes: + if subnode.name == name: + return subnode + return None + + def Scan(self): + """Scan the subnodes of a node + + This should be implemented by subclasses + """ + raise NotImplementedError() + class Fdt: """Provides simple access to a flat device tree blob. @@ -173,3 +193,40 @@ class Fdt: """ def __init__(self, fname): self._fname = fname + + def Scan(self, root='/'): + """Scan a device tree, building up a tree of Node objects + + This fills in the self._root property + + Args: + root: Ignored + + TODO(sjg@chromium.org): Implement the 'root' parameter + """ + self._root = self.Node(self, 0, '/', '/') + self._root.Scan() + + def GetRoot(self): + """Get the root Node of the device tree + + Returns: + The root Node object + """ + return self._root + + def GetNode(self, path): + """Look up a node from its path + + Args: + path: Path to look up, e.g. '/microcode/update@0' + Returns: + Node object, or None if not found + """ + node = self._root + for part in path.split('/')[1:]: + node = node._FindNode(part) + if not node: + return None + return node + diff --git a/tools/dtoc/fdt_fallback.py b/tools/dtoc/fdt_fallback.py index 84a3db1..5b0f2a1 100644 --- a/tools/dtoc/fdt_fallback.py +++ b/tools/dtoc/fdt_fallback.py @@ -81,22 +81,6 @@ class FdtFallback(Fdt): def __init__(self, fname): Fdt.__init__(self, fname) - def Scan(self): - """Scan a device tree, building up a tree of Node objects - - This fills in the self._root property - """ - self._root = Node(self, 0, '/', '/') - self._root.Scan() - - def GetRoot(self): - """Get the root Node of the device tree - - Returns: - The root Node object - """ - return self._root - def GetSubNodes(self, node): """Returns a list of sub-nodes of a given node diff --git a/tools/dtoc/fdt_normal.py b/tools/dtoc/fdt_normal.py index 6f019c1..861f60c 100644 --- a/tools/dtoc/fdt_normal.py +++ b/tools/dtoc/fdt_normal.py @@ -94,22 +94,6 @@ class FdtNormal(Fdt): """ return self._fdt - def Scan(self): - """Scan a device tree, building up a tree of Node objects - - This fills in the self._root property - """ - self._root = Node(self, 0, '/', '/') - self._root.Scan() - - def GetRoot(self): - """Get the root Node of the device tree - - Returns: - The root Node object - """ - return self._root - def GetProps(self, node): """Get all properties from a node. -- cgit v0.10.2 From 1f1864b408b74fe04d787d9f8941f7b18b9ded66 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:08 -0600 Subject: patman: Add a tools library for using temporary files For tools which want to use input files and temporary output, it is useful to have the handling of these dealt with in one place. Add a new library which allows input files to be read, and output files to be written, all based on a common directory structure. Signed-off-by: Simon Glass diff --git a/tools/patman/tools.py b/tools/patman/tools.py new file mode 100644 index 0000000..ba24853 --- /dev/null +++ b/tools/patman/tools.py @@ -0,0 +1,120 @@ +# +# Copyright (c) 2016 Google, Inc +# +# SPDX-License-Identifier: GPL-2.0+ +# + +import os +import shutil +import tempfile + +import tout + +outdir = None +indirs = None +preserve_outdir = False + +def PrepareOutputDir(dirname, preserve=False): + """Select an output directory, ensuring it exists. + + This either creates a temporary directory or checks that the one supplied + by the user is valid. For a temporary directory, it makes a note to + remove it later if required. + + Args: + dirname: a string, name of the output directory to use to store + intermediate and output files. If is None - create a temporary + directory. + preserve: a Boolean. If outdir above is None and preserve is False, the + created temporary directory will be destroyed on exit. + + Raises: + OSError: If it cannot create the output directory. + """ + global outdir, preserve_outdir + + preserve_outdir = dirname or preserve + if dirname: + outdir = dirname + if not os.path.isdir(outdir): + try: + os.makedirs(outdir) + except OSError as err: + raise CmdError("Cannot make output directory '%s': '%s'" % + (outdir, err.strerror)) + tout.Debug("Using output directory '%s'" % outdir) + else: + outdir = tempfile.mkdtemp(prefix='binman.') + tout.Debug("Using temporary directory '%s'" % outdir) + +def _RemoveOutputDir(): + global outdir + + shutil.rmtree(outdir) + tout.Debug("Deleted temporary directory '%s'" % outdir) + outdir = None + +def FinaliseOutputDir(): + global outdir, preserve_outdir + + """Tidy up: delete output directory if temporary and not preserved.""" + if outdir and not preserve_outdir: + _RemoveOutputDir() + +def GetOutputFilename(fname): + """Return a filename within the output directory. + + Args: + fname: Filename to use for new file + + Returns: + The full path of the filename, within the output directory + """ + return os.path.join(outdir, fname) + +def _FinaliseForTest(): + """Remove the output directory (for use by tests)""" + global outdir + + if outdir: + _RemoveOutputDir() + +def SetInputDirs(dirname): + """Add a list of input directories, where input files are kept. + + Args: + dirname: a list of paths to input directories to use for obtaining + files needed by binman to place in the image. + """ + global indir + + indir = dirname + tout.Debug("Using input directories %s" % indir) + +def GetInputFilename(fname): + """Return a filename for use as input. + + Args: + fname: Filename to use for new file + + Returns: + The full path of the filename, within the input directory + """ + if not indir: + return fname + for dirname in indir: + pathname = os.path.join(dirname, fname) + if os.path.exists(pathname): + return pathname + + raise ValueError("Filename '%s' not found in input path (%s)" % + (fname, ','.join(indir))) + +def Align(pos, align): + if align: + mask = align - 1 + pos = (pos + mask) & ~mask + return pos + +def NotPowerOfTwo(num): + return num and (num & (num - 1)) -- cgit v0.10.2 From 0faf6144fd1f6443a52abb0d80a6ca1238ecc029 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:09 -0600 Subject: patman: Add a library to handle logging and progress When tools want to display information of varying levels of importance, it helps to provide the user with control over the verbosity of these messages. Progress messages work best if they are displayed and then removed from the display when no-longer relevant. Add a new tout library (terminal out) to handle these tasks. Signed-off-by: Simon Glass diff --git a/tools/patman/tout.py b/tools/patman/tout.py new file mode 100644 index 0000000..c5fbd80 --- /dev/null +++ b/tools/patman/tout.py @@ -0,0 +1,166 @@ +# Copyright (c) 2016 Google, Inc +# +# SPDX-License-Identifier: GPL-2.0+ +# +# Terminal output logging. +# + +import sys + +import terminal + +# Output verbosity levels that we support +ERROR = 0 +WARNING = 1 +NOTICE = 2 +INFO = 3 +DEBUG = 4 + +""" +This class handles output of progress and other useful information +to the user. It provides for simple verbosity level control and can +output nothing but errors at verbosity zero. + +The idea is that modules set up an Output object early in their years and pass +it around to other modules that need it. This keeps the output under control +of a single class. + +Public properties: + verbose: Verbosity level: 0=silent, 1=progress, 3=full, 4=debug +""" +def __enter__(): + return + +def __exit__(unused1, unused2, unused3): + """Clean up and remove any progress message.""" + ClearProgress() + return False + +def UserIsPresent(): + """This returns True if it is likely that a user is present. + + Sometimes we want to prompt the user, but if no one is there then this + is a waste of time, and may lock a script which should otherwise fail. + + Returns: + True if it thinks the user is there, and False otherwise + """ + return stdout_is_tty and verbose > 0 + +def ClearProgress(): + """Clear any active progress message on the terminal.""" + if verbose > 0 and stdout_is_tty: + _stdout.write('\r%s\r' % (" " * len (_progress))) + _stdout.flush() + +def Progress(msg, warning=False, trailer='...'): + """Display progress information. + + Args: + msg: Message to display. + warning: True if this is a warning.""" + ClearProgress() + if verbose > 0: + _progress = msg + trailer + if stdout_is_tty: + col = _color.YELLOW if warning else _color.GREEN + _stdout.write('\r' + _color.Color(col, _progress)) + _stdout.flush() + else: + _stdout.write(_progress + '\n') + +def _Output(level, msg, color=None): + """Output a message to the terminal. + + Args: + level: Verbosity level for this message. It will only be displayed if + this as high as the currently selected level. + msg; Message to display. + error: True if this is an error message, else False. + """ + if verbose >= level: + ClearProgress() + if color: + msg = _color.Color(color, msg) + _stdout.write(msg + '\n') + +def DoOutput(level, msg): + """Output a message to the terminal. + + Args: + level: Verbosity level for this message. It will only be displayed if + this as high as the currently selected level. + msg; Message to display. + """ + _Output(level, msg) + +def Error(msg): + """Display an error message + + Args: + msg; Message to display. + """ + _Output(0, msg, _color.RED) + +def Warning(msg): + """Display a warning message + + Args: + msg; Message to display. + """ + _Output(1, msg, _color.YELLOW) + +def Notice(msg): + """Display an important infomation message + + Args: + msg; Message to display. + """ + _Output(2, msg) + +def Info(msg): + """Display an infomation message + + Args: + msg; Message to display. + """ + _Output(3, msg) + +def Debug(msg): + """Display a debug message + + Args: + msg; Message to display. + """ + _Output(4, msg) + +def UserOutput(msg): + """Display a message regardless of the current output level. + + This is used when the output was specifically requested by the user. + Args: + msg; Message to display. + """ + _Output(0, msg) + +def Init(_verbose=WARNING, stdout=sys.stdout): + """Initialize a new output object. + + Args: + verbose: Verbosity level (0-4). + stdout: File to use for stdout. + """ + global verbose, _progress, _color, _stdout, stdout_is_tty + + verbose = _verbose + _progress = '' # Our last progress message + _color = terminal.Color() + _stdout = stdout + + # TODO(sjg): Move this into Chromite libraries when we have them + stdout_is_tty = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty() + +def Uninit(): + ClearProgress() + +Init() -- cgit v0.10.2 From 355c67c35a8ce5aa9e9e2e2e8df99413c8215093 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:10 -0600 Subject: dtoc: Allow the device tree to be compiled from source If a source device tree is provide to the Fdt() constructors, compile it automatically. This will be used in tests, where we want to build a particular test .dts file and check that it works correctly in binman. Signed-off-by: Simon Glass diff --git a/tools/dtoc/fdt_fallback.py b/tools/dtoc/fdt_fallback.py index 5b0f2a1..f76f42a 100644 --- a/tools/dtoc/fdt_fallback.py +++ b/tools/dtoc/fdt_fallback.py @@ -80,6 +80,8 @@ class FdtFallback(Fdt): def __init__(self, fname): Fdt.__init__(self, fname) + if self._fname: + self._fname = fdt_util.EnsureCompiled(self._fname) def GetSubNodes(self, node): """Returns a list of sub-nodes of a given node diff --git a/tools/dtoc/fdt_normal.py b/tools/dtoc/fdt_normal.py index 861f60c..d9ba4ac 100644 --- a/tools/dtoc/fdt_normal.py +++ b/tools/dtoc/fdt_normal.py @@ -83,8 +83,11 @@ class FdtNormal(Fdt): """ def __init__(self, fname): Fdt.__init__(self, fname) - with open(self._fname) as fd: - self._fdt = fd.read() + if self._fname: + self._fname = fdt_util.EnsureCompiled(self._fname) + + with open(self._fname) as fd: + self._fdt = fd.read() def GetFdt(self): """Get the contents of the FDT diff --git a/tools/dtoc/fdt_util.py b/tools/dtoc/fdt_util.py index 6b57248..3e25a8b 100644 --- a/tools/dtoc/fdt_util.py +++ b/tools/dtoc/fdt_util.py @@ -6,7 +6,12 @@ # SPDX-License-Identifier: GPL-2.0+ # +import os import struct +import tempfile + +import command +import tools def fdt32_to_cpu(val): """Convert a device tree cell to an integer @@ -18,3 +23,39 @@ def fdt32_to_cpu(val): A native-endian integer value """ return struct.unpack(">I", val)[0] + +def EnsureCompiled(fname): + """Compile an fdt .dts source file into a .dtb binary blob if needed. + + Args: + fname: Filename (if .dts it will be compiled). It not it will be + left alone + + Returns: + Filename of resulting .dtb file + """ + _, ext = os.path.splitext(fname) + if ext != '.dts': + return fname + + dts_input = tools.GetOutputFilename('source.dts') + dtb_output = tools.GetOutputFilename('source.dtb') + + search_paths = [os.path.join(os.getcwd(), 'include')] + root, _ = os.path.splitext(fname) + args = ['-E', '-P', '-x', 'assembler-with-cpp', '-D__ASSEMBLY__'] + args += ['-Ulinux'] + for path in search_paths: + args.extend(['-I', path]) + args += ['-o', dts_input, fname] + command.Run('cc', *args) + + # If we don't have a directory, put it in the tools tempdir + search_list = [] + for path in search_paths: + search_list.extend(['-i', path]) + args = ['-I', 'dts', '-o', dtb_output, '-O', 'dtb'] + args.extend(search_list) + args.append(dts_input) + command.Run('dtc', *args) + return dtb_output -- cgit v0.10.2 From 6b93c55f59aba3a3fff36b771ca75739e31ba7db Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:11 -0600 Subject: dtoc: Drop the convert_dash parameter to GetProps() This is not used anywhere in dtoc, so drop it. Signed-off-by: Simon Glass diff --git a/tools/dtoc/fdt_fallback.py b/tools/dtoc/fdt_fallback.py index f76f42a..1c8c9c7 100644 --- a/tools/dtoc/fdt_fallback.py +++ b/tools/dtoc/fdt_fallback.py @@ -98,12 +98,11 @@ class FdtFallback(Fdt): out = command.Output('fdtget', self._fname, '-l', node) return out.strip().splitlines() - def GetProps(self, node, convert_dashes=False): + def GetProps(self, node): """Get all properties from a node Args: node: full path to node name to look in - convert_dashes: True to convert - to _ in node names Returns: A dictionary containing all the properties, indexed by node name. @@ -118,8 +117,6 @@ class FdtFallback(Fdt): props_dict = {} for prop in props: name = prop - if convert_dashes: - prop = re.sub('-', '_', prop) props_dict[prop] = self.GetProp(node, name) return props_dict -- cgit v0.10.2 From 346179f0d3383e88f7df117b13820df70f68c74a Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:12 -0600 Subject: dtoc: Prepare for supporting changing of device trees For binman we need to support deleting properties in the device tree. This will change the offsets of nodes after the deletion. In preparation, add code to keep track of when the offsets are invalid, and regenerate them. Signed-off-by: Simon Glass diff --git a/tools/dtoc/fdt_normal.py b/tools/dtoc/fdt_normal.py index d9ba4ac..4a667a1 100644 --- a/tools/dtoc/fdt_normal.py +++ b/tools/dtoc/fdt_normal.py @@ -53,15 +53,24 @@ class Node(NodeBase): def __init__(self, fdt, offset, name, path): NodeBase.__init__(self, fdt, offset, name, path) + def Offset(self): + """Returns the offset of a node, after checking the cache + + This should be used instead of self._offset directly, to ensure that + the cache does not contain invalid offsets. + """ + self._fdt.CheckCache() + return self._offset + def Scan(self): """Scan a node's properties and subnodes This fills in the props and subnodes properties, recursively searching into subnodes so that the entire tree is built. """ - self.props = self._fdt.GetProps(self.path) + self.props = self._fdt.GetProps(self, self.path) - offset = libfdt.fdt_first_subnode(self._fdt.GetFdt(), self._offset) + offset = libfdt.fdt_first_subnode(self._fdt.GetFdt(), self.Offset()) while offset >= 0: sep = '' if self.path[-1] == '/' else '/' name = libfdt.Name(self._fdt.GetFdt(), offset) @@ -72,6 +81,19 @@ class Node(NodeBase): node.Scan() offset = libfdt.fdt_next_subnode(self._fdt.GetFdt(), offset) + def Refresh(self, my_offset): + """Fix up the _offset for each node, recursively + + Note: This does not take account of property offsets - these will not + be updated. + """ + if self._offset != my_offset: + #print '%s: %d -> %d\n' % (self.path, self._offset, my_offset) + self._offset = my_offset + offset = libfdt.fdt_first_subnode(self._fdt.GetFdt(), self._offset) + for subnode in self.subnodes: + subnode.Refresh(offset) + offset = libfdt.fdt_next_subnode(self._fdt.GetFdt(), offset) class FdtNormal(Fdt): """Provides simple access to a flat device tree blob using libfdt. @@ -83,6 +105,7 @@ class FdtNormal(Fdt): """ def __init__(self, fname): Fdt.__init__(self, fname) + self._cached_offsets = False if self._fname: self._fname = fdt_util.EnsureCompiled(self._fname) @@ -97,7 +120,7 @@ class FdtNormal(Fdt): """ return self._fdt - def GetProps(self, node): + def GetProps(self, node, path): """Get all properties from a node. Args: @@ -110,7 +133,7 @@ class FdtNormal(Fdt): Raises: ValueError: if the node does not exist. """ - offset = libfdt.fdt_path_offset(self._fdt, node) + offset = libfdt.fdt_path_offset(self._fdt, path) if offset < 0: libfdt.Raise(offset) props_dict = {} @@ -124,6 +147,21 @@ class FdtNormal(Fdt): poffset = libfdt.fdt_next_property_offset(self._fdt, poffset) return props_dict + def Invalidate(self): + """Mark our offset cache as invalid""" + self._cached_offsets = False + + def CheckCache(self): + """Refresh the offset cache if needed""" + if self._cached_offsets: + return + self.Refresh() + self._cached_offsets = True + + def Refresh(self): + """Refresh the offset cache""" + self._root.Refresh(0) + @classmethod def Node(self, fdt, offset, name, path): """Create a new node -- cgit v0.10.2 From 0170804f60b19a2033ac39964fcd192a0c7eda42 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:13 -0600 Subject: dtoc: Move to using bytearray Since we want to be able to change the in-memory device tree using libfdt, use a bytearray instead of a string. This makes interfacing from Python easier. Signed-off-by: Simon Glass diff --git a/lib/libfdt/libfdt.swig b/lib/libfdt/libfdt.swig index 14f583d..26d42fc 100644 --- a/lib/libfdt/libfdt.swig +++ b/lib/libfdt/libfdt.swig @@ -75,6 +75,14 @@ struct fdt_property { } %} +%typemap(in) (const void *) { + if (!PyByteArray_Check($input)) { + SWIG_exception_fail(SWIG_TypeError, "in method '" "$symname" "', argument " + "$argnum"" of type '" "$type""'"); + } + $1 = (void *) PyByteArray_AsString($input); +} + const void *fdt_offset_ptr(const void *fdt, int offset, unsigned int checklen); int fdt_path_offset(const void *fdt, const char *path); int fdt_first_property_offset(const void *fdt, int nodeoffset); diff --git a/tools/dtoc/fdt_normal.py b/tools/dtoc/fdt_normal.py index 4a667a1..eb45742 100644 --- a/tools/dtoc/fdt_normal.py +++ b/tools/dtoc/fdt_normal.py @@ -110,7 +110,7 @@ class FdtNormal(Fdt): self._fname = fdt_util.EnsureCompiled(self._fname) with open(self._fname) as fd: - self._fdt = fd.read() + self._fdt = bytearray(fd.read()) def GetFdt(self): """Get the contents of the FDT -- cgit v0.10.2 From 2a70d897ed68fd521411a10831ac05e1ffdd3d41 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:14 -0600 Subject: dtoc: Support deleting device tree properties Add support for deleting a device tree property. With the fallback implementation this uses fdtput. With libfdt it uses the API call and updates the offsets afterwards. Signed-off-by: Simon Glass diff --git a/lib/libfdt/libfdt.swig b/lib/libfdt/libfdt.swig index 26d42fc..ce516fd 100644 --- a/lib/libfdt/libfdt.swig +++ b/lib/libfdt/libfdt.swig @@ -95,3 +95,15 @@ const char *fdt_get_name(const void *fdt, int nodeoffset, int *OUTPUT); const char *fdt_string(const void *fdt, int stroffset); int fdt_first_subnode(const void *fdt, int offset); int fdt_next_subnode(const void *fdt, int offset); + +%typemap(in) (void *) { + if (!PyByteArray_Check($input)) { + SWIG_exception_fail(SWIG_TypeError, "in method '" "$symname" "', argument " + "$argnum"" of type '" "$type""'"); + } + $1 = PyByteArray_AsString($input); +} + +int fdt_delprop(void *fdt, int nodeoffset, const char *name); + +const char *fdt_strerror(int errval); diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py index c0ce5af..f01c7b1 100644 --- a/tools/dtoc/fdt.py +++ b/tools/dtoc/fdt.py @@ -184,6 +184,16 @@ class NodeBase: """ raise NotImplementedError() + def DeleteProp(self, prop_name): + """Delete a property of a node + + This should be implemented by subclasses + + Args: + prop_name: Name of the property to delete + """ + raise NotImplementedError() + class Fdt: """Provides simple access to a flat device tree blob. diff --git a/tools/dtoc/fdt_fallback.py b/tools/dtoc/fdt_fallback.py index 1c8c9c7..0c0ebbc 100644 --- a/tools/dtoc/fdt_fallback.py +++ b/tools/dtoc/fdt_fallback.py @@ -70,6 +70,19 @@ class Node(NodeBase): node.Scan() + def DeleteProp(self, prop_name): + """Delete a property of a node + + The property is deleted using fdtput. + + Args: + prop_name: Name of the property to delete + Raises: + CommandError if the property does not exist + """ + args = [self._fdt._fname, '-d', self.path, prop_name] + command.Output('fdtput', *args) + del self.props[prop_name] class FdtFallback(Fdt): """Provides simple access to a flat device tree blob using fdtget/fdtput diff --git a/tools/dtoc/fdt_normal.py b/tools/dtoc/fdt_normal.py index eb45742..52d8055 100644 --- a/tools/dtoc/fdt_normal.py +++ b/tools/dtoc/fdt_normal.py @@ -20,6 +20,11 @@ import libfdt # This implementation uses a libfdt Python library to access the device tree, # so it is fairly efficient. +def CheckErr(errnum, msg): + if errnum: + raise ValueError('Error %d: %s: %s' % + (errnum, libfdt.fdt_strerror(errnum), msg)) + class Prop(PropBase): """A device tree property @@ -95,6 +100,21 @@ class Node(NodeBase): subnode.Refresh(offset) offset = libfdt.fdt_next_subnode(self._fdt.GetFdt(), offset) + def DeleteProp(self, prop_name): + """Delete a property of a node + + The property is deleted and the offset cache is invalidated. + + Args: + prop_name: Name of the property to delete + Raises: + ValueError if the property does not exist + """ + CheckErr(libfdt.fdt_delprop(self._fdt.GetFdt(), self.Offset(), prop_name), + "Node '%s': delete property: '%s'" % (self.path, prop_name)) + del self.props[prop_name] + self._fdt.Invalidate() + class FdtNormal(Fdt): """Provides simple access to a flat device tree blob using libfdt. -- cgit v0.10.2 From da5f74998b9e5e6b706608a4ca625ef0ee195150 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:15 -0600 Subject: dtoc: Support packing the device tree After any node/property deletion the device tree can be packed to remove spare space. Add a way to perform this operation. Note that for fdt_fallback, fdtput automatically packs the device tree after deletion, so no action is required here. Signed-off-by: Simon Glass diff --git a/lib/libfdt/libfdt.swig b/lib/libfdt/libfdt.swig index ce516fd..0cb7977 100644 --- a/lib/libfdt/libfdt.swig +++ b/lib/libfdt/libfdt.swig @@ -107,3 +107,4 @@ int fdt_next_subnode(const void *fdt, int offset); int fdt_delprop(void *fdt, int nodeoffset, const char *name); const char *fdt_strerror(int errval); +int fdt_pack(void *fdt); diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py index f01c7b1..403eb1f 100644 --- a/tools/dtoc/fdt.py +++ b/tools/dtoc/fdt.py @@ -240,3 +240,19 @@ class Fdt: return None return node + def Flush(self): + """Flush device tree changes back to the file + + If the device tree has changed in memory, write it back to the file. + Subclasses can implement this if needed. + """ + pass + + def Pack(self): + """Pack the device tree down to its minimum size + + When nodes and properties shrink or are deleted, wasted space can + build up in the device tree binary. Subclasses can implement this + to remove that spare space. + """ + pass diff --git a/tools/dtoc/fdt_normal.py b/tools/dtoc/fdt_normal.py index 52d8055..f2cf608 100644 --- a/tools/dtoc/fdt_normal.py +++ b/tools/dtoc/fdt_normal.py @@ -140,6 +140,17 @@ class FdtNormal(Fdt): """ return self._fdt + def Flush(self): + """Flush device tree changes back to the file""" + with open(self._fname, 'wb') as fd: + fd.write(self._fdt) + + def Pack(self): + """Pack the device tree down to its minimum size""" + CheckErr(libfdt.fdt_pack(self._fdt), 'pack') + fdt_len = libfdt.fdt_totalsize(self._fdt) + del self._fdt[fdt_len:] + def GetProps(self, node, path): """Get all properties from a node. -- cgit v0.10.2 From babdbde68f1b993289462394f209f4010c761246 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:16 -0600 Subject: dtoc: Support finding the offset of a property Add a way to find the byte offset of a property within the device tree. This is only supported with the normal libfdt implementation since fdtget does not provide this information. Signed-off-by: Simon Glass diff --git a/lib/libfdt/libfdt.swig b/lib/libfdt/libfdt.swig index 0cb7977..b24c72b 100644 --- a/lib/libfdt/libfdt.swig +++ b/lib/libfdt/libfdt.swig @@ -108,3 +108,6 @@ int fdt_delprop(void *fdt, int nodeoffset, const char *name); const char *fdt_strerror(int errval); int fdt_pack(void *fdt); + +int fdt_totalsize(const void *fdt); +int fdt_off_dt_struct(const void *fdt); diff --git a/tools/dtoc/fdt.py b/tools/dtoc/fdt.py index 403eb1f..816fdbe 100644 --- a/tools/dtoc/fdt.py +++ b/tools/dtoc/fdt.py @@ -144,6 +144,17 @@ class PropBase: else: return True + def GetOffset(self): + """Get the offset of a property + + This can be implemented by subclasses. + + Returns: + The offset of the property (struct fdt_property) within the + file, or None if not known. + """ + return None + class NodeBase: """A device tree node diff --git a/tools/dtoc/fdt_normal.py b/tools/dtoc/fdt_normal.py index f2cf608..aae258e 100644 --- a/tools/dtoc/fdt_normal.py +++ b/tools/dtoc/fdt_normal.py @@ -43,6 +43,14 @@ class Prop(PropBase): return self.type, self.value = self.BytesToValue(bytes) + def GetOffset(self): + """Get the offset of a property + + Returns: + The offset of the property (struct fdt_property) within the file + """ + return self._node._fdt.GetStructOffset(self._offset) + class Node(NodeBase): """A device tree node @@ -193,6 +201,16 @@ class FdtNormal(Fdt): """Refresh the offset cache""" self._root.Refresh(0) + def GetStructOffset(self, offset): + """Get the file offset of a given struct offset + + Args: + offset: Offset within the 'struct' region of the device tree + Returns: + Position of @offset within the device tree binary + """ + return libfdt.fdt_off_dt_struct(self._fdt) + offset + @classmethod def Node(self, fdt, offset, name, path): """Create a new node -- cgit v0.10.2 From 20024daee58906712f71b927bd86951d1ddb469d Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:17 -0600 Subject: dtoc: Correct quotes in fdt_util The style is to use single quotes for strings where possible. Adjust this function. Signed-off-by: Simon Glass diff --git a/tools/dtoc/fdt_util.py b/tools/dtoc/fdt_util.py index 3e25a8b..32f41d7 100644 --- a/tools/dtoc/fdt_util.py +++ b/tools/dtoc/fdt_util.py @@ -22,7 +22,7 @@ def fdt32_to_cpu(val): Return: A native-endian integer value """ - return struct.unpack(">I", val)[0] + return struct.unpack('>I', val)[0] def EnsureCompiled(fname): """Compile an fdt .dts source file into a .dtb binary blob if needed. -- cgit v0.10.2 From 8f224b3734d042884a8981a14db64c48e87b87a2 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 25 Jul 2016 18:59:18 -0600 Subject: dtoc: Add methods for reading data from properties Provide easy helpers for reading integer, string and boolean values from device-tree properties. Signed-off-by: Simon Glass diff --git a/tools/dtoc/fdt_util.py b/tools/dtoc/fdt_util.py index 32f41d7..3a10838 100644 --- a/tools/dtoc/fdt_util.py +++ b/tools/dtoc/fdt_util.py @@ -59,3 +59,28 @@ def EnsureCompiled(fname): args.append(dts_input) command.Run('dtc', *args) return dtb_output + +def GetInt(node, propname, default=None): + prop = node.props.get(propname) + if not prop: + return default + value = fdt32_to_cpu(prop.value) + if type(value) == type(list): + raise ValueError("Node '%s' property '%' has list value: expecting" + "a single integer" % (node.name, propname)) + return value + +def GetString(node, propname, default=None): + prop = node.props.get(propname) + if not prop: + return default + value = prop.value + if type(value) == type(list): + raise ValueError("Node '%s' property '%' has list value: expecting" + "a single string" % (node.name, propname)) + return value + +def GetBool(node, propname, default=False): + if propname in node.props: + return True + return default -- cgit v0.10.2