Skip to content

Commit

Permalink
bitbake: data: Add sup0port for new BB_HASH_CODEPARSER_VALS for cache…
Browse files Browse the repository at this point in the history 
… optimisation

Currently the codeparser cache ends up being extended for every parse run
since there are values in the functions such as the result of os.getpid()
from LOGFIFO in OE-Core.

Digging into that issue, there are also lots of similar but different
functions being parsed where the change might just be a path to WORKDIR,
a change in PN or PV or something like DATE/TIME.

There is no reason we have to use these changing values when computing the
dependenies of the functions. Even with a small tweak like:

BB_HASH_CODEPARSER_VALS = "LOGFIFO=/ T=/ WORKDIR=/ DATE=1234 TIME=1234 PV=0.0-1 PN=nopn"

the cache is reduced from ~4.6MB, increasing by ~300kb for every parse run
to around 1.3MB and remaining static for oe-core and meta-oe. In my local
build, admittedly heavily experimented with, the cache had grown to 120MB.

The benefits of doing this are:

 * faster load time for bitbake since the cache is smaller to read from disk
   and load into memory
 * being able to skip saving the cache upon shutdown
 * lower memory footprint for bitbake
 * faster codeparser data lookups (since there is less data to search)

We only use these special values when passing code fragments to the codeparser
to parse so the real variable values should otherwise be used in the hash data.

The overall effect of this change, combined with others to avoid saving unchanged
cache files can be ~2s on a ~16s parse on my local system and results in a more
responsive feeling bitbake. It also allows parsing performance to be investigated
more consistently.

(Bitbake rev: e584799386da3fee3083d8d724c8b6f611852aa9)

Signed-off-by: Richard Purdie <[email protected]>
Signed-off-by: Alexandre Belloni <[email protected]>
  • Loading branch information
@rpurdie @alexandrebelloni
rpurdie authored and alexandrebelloni committed Jan 24, 2023
1 parent 8f3cce3 commit 5b4836c
Show file tree
Hide file tree
Showing 2 changed files with 17 additions and 12 deletions.
15 changes: 10 additions & 5 deletions bitbake/lib/bb/data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -261,7 +261,7 @@ def write_func(func, o, call = False):
newdeps |= set((d.getVarFlag(dep, "vardeps") or "").split())
newdeps -= seen

def build_dependencies(key, keys, mod_funcs, shelldeps, varflagsexcl, ignored_vars, d):
def build_dependencies(key, keys, mod_funcs, shelldeps, varflagsexcl, ignored_vars, d, codeparsedata):
def handle_contains(value, contains, exclusions, d):
newvalue = []
if value:
Expand Down Expand Up @@ -312,14 +312,14 @@ def handle_remove(value, deps, removes, d):
value = varflags.get("vardepvalue")
elif varflags.get("func"):
if varflags.get("python"):
value = d.getVarFlag(key, "_content", False)
value = codeparsedata.getVarFlag(key, "_content", False)
parser = bb.codeparser.PythonParser(key, logger)
parser.parse_python(value, filename=varflags.get("filename"), lineno=varflags.get("lineno"))
deps = deps | parser.references
deps = deps | (keys & parser.execs)
value = handle_contains(value, parser.contains, exclusions, d)
else:
value, parsedvar = d.getVarFlag(key, "_content", False, retparser=True)
value, parsedvar = codeparsedata.getVarFlag(key, "_content", False, retparser=True)
parser = bb.codeparser.ShellParser(key, logger)
parser.parse_shell(parsedvar.value)
deps = deps | shelldeps
Expand Down Expand Up @@ -378,12 +378,17 @@ def generate_dependencies(d, ignored_vars):
shelldeps = set(key for key in d.getVar("__exportlist", False) if d.getVarFlag(key, "export", False) and not d.getVarFlag(key, "unexport", False))
varflagsexcl = d.getVar('BB_SIGNATURE_EXCLUDE_FLAGS')

codeparserd = d.createCopy()
for forced in (d.getVar('BB_HASH_CODEPARSER_VALS') or "").split():
key, value = forced.split("=", 1)
codeparserd.setVar(key, value)

deps = {}
values = {}

tasklist = d.getVar('__BBTASKS', False) or []
for task in tasklist:
deps[task], values[task] = build_dependencies(task, keys, mod_funcs, shelldeps, varflagsexcl, ignored_vars, d)
deps[task], values[task] = build_dependencies(task, keys, mod_funcs, shelldeps, varflagsexcl, ignored_vars, d, codeparserd)
newdeps = deps[task]
seen = set()
while newdeps:
Expand All @@ -392,7 +397,7 @@ def generate_dependencies(d, ignored_vars):
newdeps = set()
for dep in nextdeps:
if dep not in deps:
deps[dep], values[dep] = build_dependencies(dep, keys, mod_funcs, shelldeps, varflagsexcl, ignored_vars, d)
deps[dep], values[dep] = build_dependencies(dep, keys, mod_funcs, shelldeps, varflagsexcl, ignored_vars, d, codeparserd)
newdeps |= deps[dep]
newdeps -= seen
#print "For %s: %s" % (task, str(deps[task]))
Expand Down
14 changes: 7 additions & 7 deletions bitbake/lib/bb/tests/codeparser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -318,7 +318,7 @@ def test_python(self):
"filename": "example.bb",
})

deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d)
deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)

self.assertEqual(deps, set(["somevar", "bar", "something", "inexpand", "test", "test2", "a"]))

Expand Down Expand Up @@ -365,7 +365,7 @@ def test_shell(self):
self.d.setVarFlags("FOO", {"func": True})
self.setEmptyVars(execs)

deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d)
deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)

self.assertEqual(deps, set(["somevar", "inverted"] + execs))

Expand All @@ -375,7 +375,7 @@ def test_vardeps(self):
self.d.setVar("FOO", "foo=oe_libinstall; eval $foo")
self.d.setVarFlag("FOO", "vardeps", "oe_libinstall")

deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d)
deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)

self.assertEqual(deps, set(["oe_libinstall"]))

Expand All @@ -384,7 +384,7 @@ def test_vardeps_expand(self):
self.d.setVar("FOO", "foo=oe_libinstall; eval $foo")
self.d.setVarFlag("FOO", "vardeps", "${@'oe_libinstall'}")

deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d)
deps, values = bb.data.build_dependencies("FOO", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)

self.assertEqual(deps, set(["oe_libinstall"]))

Expand All @@ -399,7 +399,7 @@ def test_contains_vardeps(self):
# Check dependencies
self.d.setVar('ANOTHERVAR', expr)
self.d.setVar('TESTVAR', 'anothervalue testval testval2')
deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(), self.d)
deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
self.assertEqual(sorted(values.splitlines()),
sorted([expr,
'TESTVAR{anothervalue} = Set',
Expand All @@ -418,14 +418,14 @@ def test_contains_vardeps_excluded(self):
self.d.setVar('ANOTHERVAR', varval)
self.d.setVar('TESTVAR', 'anothervalue testval testval2')
self.d.setVar('TESTVAR2', 'testval3')
deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(["TESTVAR"]), self.d)
deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(["TESTVAR"]), self.d, self.d)
self.assertEqual(sorted(values.splitlines()), sorted([varval]))
self.assertEqual(deps, set(["TESTVAR2"]))
self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue'])

# Check the vardepsexclude flag is handled by contains functionality
self.d.setVarFlag('ANOTHERVAR', 'vardepsexclude', 'TESTVAR')
deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(), self.d)
deps, values = bb.data.build_dependencies("ANOTHERVAR", set(self.d.keys()), set(), set(), set(), set(), self.d, self.d)
self.assertEqual(sorted(values.splitlines()), sorted([varval]))
self.assertEqual(deps, set(["TESTVAR2"]))
self.assertEqual(self.d.getVar('ANOTHERVAR').split(), ['testval3', 'anothervalue'])
Expand Down

0 comments on commit 5b4836c

Please sign in to comment.