Bug 15069

This is a bit of weird one.

I wanted to use PyCryptodome (https://pypi.org/project/pycryptodome/) from my recipe, but importing the module would make the build fail with a very cryptic error. For example, given I have a recipe "foo-test_0.1.bb" with content:

```
DESCRIPTION = "Test"
LICENSE = "CLOSED"

SSTATETASKS += "do_foo"

python do_foo() {
    import Crypto.Cipher.AES
}

addtask do_foo before do_install
```

When you run "bitbake foo-test", then you get the following error:

```
ERROR: foo-test-0.1-r0 do_foo: Error during parse shell code, the last 5 lines are:
        else
                touch /tmp/test-yocto-master/poky/build/sstate-cache/0f/df/sstate:foo-test:core2-64-poky-linux:0.1:r0:core2-64:10:0fdf24a05e517aecf93d535478102c428125efda6bb3b3a681020cc6a900eb85_foo.tar.zst 2>/dev/null || true
        fi
        rm $TFILE

ERROR: foo-test-0.1-r0 do_foo: Error executing a python function in exec_func_python() autogenerated:

The stack trace of python calls that resulted in this exception/failure was:
File: 'exec_func_python() autogenerated', lineno: 2, function: <module>
     0001:
 *** 0002:sstate_task_postfunc(d)
     0003:
File: '/tmp/test-yocto-master/poky/meta/classes-global/sstate.bbclass', lineno: 824, function: sstate_task_postfunc
     0820:
     0821:    omask = os.umask(0o002)
     0822:    if omask != 0o002:
     0823:       bb.note("Using umask 0o002 (not %0o) for sstate packaging" % omask)
 *** 0824:    sstate_package(shared_state, d)
     0825:    os.umask(omask)
     0826:
     0827:    sstateinst = d.getVar("SSTATE_INSTDIR")
     0828:    d.setVar('SSTATE_FIXMEDIR', shared_state['fixmedir'])
File: '/tmp/test-yocto-master/poky/meta/classes-global/sstate.bbclass', lineno: 733, function: sstate_package
     0729:    for f in (d.getVar('SSTATECREATEFUNCS') or '').split() + \
     0730:             sstate_create_package + \
     0731:             (d.getVar('SSTATEPOSTCREATEFUNCS') or '').split():
     0732:        # All hooks should run in SSTATE_BUILDDIR.
 *** 0733:        bb.build.exec_func(f, d, (sstatebuild,))
     0734:
     0735:    # SSTATE_PKG may have been changed by sstate_report_unihash
     0736:    siginfo = d.getVar('SSTATE_PKG') + ".siginfo"
     0737:    if not os.path.exists(siginfo):
File: '/tmp/test-yocto-master/poky/bitbake/lib/bb/build.py', lineno: 257, function: exec_func
     0253:    with bb.utils.fileslocked(lockfiles):
     0254:        if ispython:
     0255:            exec_func_python(func, d, runfile, cwd=adir)
     0256:        else:
 *** 0257:            exec_func_shell(func, d, runfile, cwd=adir)
     0258:
     0259:    try:
     0260:        curcwd = os.getcwd()
     0261:    except:
File: '/tmp/test-yocto-master/poky/bitbake/lib/bb/build.py', lineno: 422, function: exec_func_shell
     0418:
     0419:    with open(runfile, 'w') as script:
     0420:        script.write(shell_trap_code())
     0421:
 *** 0422:        bb.data.emit_func(func, script, d)
     0423:
     0424:        if verboseShellLogging or bb.utils.to_boolean(d.getVar("BB_VERBOSE_LOGS", False)):
     0425:            script.write("set -x\n")
     0426:        if cwd:
File: '/tmp/test-yocto-master/poky/bitbake/lib/bb/data.py', lineno: 215, function: emit_func
     0211:        emit_var(key, o, d, False)
     0212:
     0213:    o.write('\n')
     0214:    emit_var(func, o, d, False) and o.write('\n')
 *** 0215:    newdeps = bb.codeparser.ShellParser(func, logger).parse_shell(d.getVar(func))
     0216:    newdeps |= set((d.getVarFlag(func, "vardeps") or "").split())
     0217:    seen = set()
     0218:    while newdeps:
     0219:        deps = newdeps
File: '/tmp/test-yocto-master/poky/bitbake/lib/bb/codeparser.py', lineno: 400, function: parse_shell
     0396:
     0397:        # Need to parse so take the hit on the real log buffer
     0398:        self.log = BufferedLogger('BitBake.Data.%s' % self._name, logging.DEBUG, self._log)
     0399:
 *** 0400:        self._parse_shell(value)
     0401:        self.execs = set(cmd for cmd in self.allexecs if cmd not in self.funcdefs)
     0402:
     0403:        codeparsercache.shellcacheextras[h] = codeparsercache.newShellCacheLine(self.execs)
     0404:
File: '/tmp/test-yocto-master/poky/bitbake/lib/bb/codeparser.py', lineno: 409, function: _parse_shell
     0405:        return self.execs
     0406:
     0407:    def _parse_shell(self, value):
     0408:        try:
 *** 0409:            tokens, _ = pyshyacc.parse(value, eof=True, debug=False)
     0410:        except Exception:
     0411:            bb.error('Error during parse shell code, the last 5 lines are:\n%s' % '\n'.join(value.split('\n')[-5:]))
     0412:            raise
     0413:
File: '/tmp/test-yocto-master/poky/bitbake/lib/bb/pysh/pyshyacc.py', lineno: 677, function: parse
     0673:    if lexer.is_empty():
     0674:        return [], remaining
     0675:    if debug:
     0676:        debug = 2
 *** 0677:    return yacc.parse(lexer=lexer, debug=debug), remaining
     0678:
     0679:#-------------------------------------------------------------------------------
     0680:# AST rendering helpers
     0681:#-------------------------------------------------------------------------------
File: '/tmp/test-yocto-master/poky/bitbake/lib/ply/yacc.py', lineno: 267, function: parse
     0263:            return self.parsedebug(input,lexer,debug,tracking,tokenfunc)
     0264:        elif tracking:
     0265:            return self.parseopt(input,lexer,debug,tracking,tokenfunc)
     0266:        else:
 *** 0267:            return self.parseopt_notrack(input,lexer,debug,tracking,tokenfunc)
     0268:
     0269:
     0270:    # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
     0271:    # parsedebug().
File: '/tmp/test-yocto-master/poky/bitbake/lib/ply/yacc.py', lineno: 1049, function: parseopt_notrack
     1045:                        token = get_token
     1046:                        restart = self.restart
     1047:                        if errtoken and not hasattr(errtoken,'lexer'):
     1048:                            errtoken.lexer = lexer
 *** 1049:                        tok = self.errorfunc(errtoken)
     1050:                        del errok, token, restart   # Delete special functions
     1051:
     1052:                        if self.errorok:
     1053:                            # User must have done some kind of panic
File: '/usr/lib/python3/dist-packages/pycparser/c_parser.py', lineno: 1858, function: p_error
     1854:        # If error recovery is added here in the future, make sure
     1855:        # _get_yacc_lookahead_token still works!
     1856:        #
     1857:        if p:
 *** 1858:            self._parse_error(
     1859:                'before: %s' % p.value,
     1860:                self._coord(lineno=p.lineno,
     1861:                            column=self.clex.find_tok_column(p)))
     1862:        else:
File: '/usr/lib/python3/dist-packages/pycparser/plyparser.py', lineno: 67, function: _parse_error
     0063:        column = (p.lexpos(token_idx) - (last_cr))
     0064:        return self._coord(p.lineno(token_idx), column)
     0065:
     0066:    def _parse_error(self, msg, coord):
 *** 0067:        raise ParseError("%s: %s" % (coord, msg))
     0068:
     0069:
     0070:def parameterized(*params):
     0071:    """ Decorator to create parameterized rules.
Exception: pycparser.plyparser.ParseError: <cdef source string>:0:1: before:


ERROR: Logfile of failure stored in: /tmp/test-yocto-master/poky/build/tmp/work/core2-64-poky-linux/foo-test/0.1-r0/temp/log.do_foo.431047
ERROR: Task (/tmp/test-yocto-master/poky/meta-mylayer/recipes-example/test/foo-test_0.1.bb:do_foo) failed with exit code '1'
```

(Note that to reproduce the issue, you might have to have both the PyCryptodome and cffi python packages installed).

The issue is that, when "import Crypto.Cipher.AES" is executed, it ultimately calls "yacc.yacc(...)" on bitbake's lib/ply/yacc.py module, because PyCryptodome uses cffi, which uses pycparser, which uses ply. And then, given that ply/yacc.py module has global state, the state is changed, which is something bitbake doesn't expect, and ultimately when bitbake tries parsing shell code at some sstate-related stages, it fails.

I don't fully understand the issue but that's the gist of it. It happens on Kirkstone and later, but it didn't happen on Dunfell.

As a workaround, I simply decided to wrap some of my logic in a separate program, so it wouldn't mess with bitbake's python process state.

Maybe having a better error message would already be something interesting.