When the SRC_URI is very long or the PREMIRROR/MIRROR is in a very deep directory, the fetcher of git, svn and others will covert the path into the filename, for example, path.replace('/', '.'), if len(path) > NAME_MAX, then we will get the "filename too long" error, the oe-core supports to build when len(TMPDIR) <= 410, but the NAME_MAX is usually 256, so we can easily get this if we put the PREMIRROR/MIRROR in a deep dir, this patch seems works, will send the PR to the mailing list later: diff --git a/bitbake/lib/bb/fetch2/__init__.py b/bitbake/lib/bb/fetch2/__init__.py index 451d104..eeb55e5 100644 --- a/bitbake/lib/bb/fetch2/__init__.py +++ b/bitbake/lib/bb/fetch2/__init__.py @@ -959,6 +959,24 @@ def get_file_checksums(filelist, pn): checksums.sort(key=operator.itemgetter(1)) return checksums +def truncate_to_name_max(dirpath, filename): + """ + Truncate the filename to meet len(filename) <= NAME_MAX -10, return + the last NAME_MAX - 10 characters if it doesn't, the -10 is for the + '.done', '.lock' and perhaps others in the future. + """ + if not os.path.exists(dirpath): + bb.utils.mkdirhier(dirpath) + + name_max = os.pathconf(dirpath, 'PC_NAME_MAX') + new_max = name_max - 10 + if len(filename) > new_max: + new_name = filename[-new_max:] + logger.debug(1, "Truncate %s to %s to meet %s (NAME_MAX - 10) characters" % \ + (filename, new_name, new_max)) + filename = new_name + + return filename class FetchData(object): """ @@ -1391,7 +1409,6 @@ class Fetch(object): # early checksum verify, so that if checksum mismatched, # fetcher still have chance to fetch from mirror update_stamp(u, ud, self.d) - except bb.fetch2.NetworkAccess: raise diff --git a/bitbake/lib/bb/fetch2/bzr.py b/bitbake/lib/bb/fetch2/bzr.py index 5d9e5f9..f91c7ee 100644 --- a/bitbake/lib/bb/fetch2/bzr.py +++ b/bitbake/lib/bb/fetch2/bzr.py @@ -51,6 +51,8 @@ class Bzr(FetchMethod): ud.revision = self.latest_revision(ud.url, ud, d) ud.localfile = data.expand('bzr_%s_%s_%s.tar.gz' % (ud.host, ud.path.replace('/', '.'), ud.revision), d) + # The length of the filename should be less than NAME_MAX - 10 + ud.localfile = bb.fetch2.truncate_to_name_max(ud.pkgdir, ud.localfile) def _buildbzrcommand(self, ud, d, command): """ diff --git a/bitbake/lib/bb/fetch2/git.py b/bitbake/lib/bb/fetch2/git.py index 6175e4c..19598ef 100644 --- a/bitbake/lib/bb/fetch2/git.py +++ b/bitbake/lib/bb/fetch2/git.py @@ -135,9 +135,15 @@ class Git(FetchMethod): if ud.rebaseable: for name in ud.names: gitsrcname = gitsrcname + '_' + ud.revisions[name] + ud.mirrortarball = 'git2_%s.tar.gz' % (gitsrcname) + # The length of the filename should be less than NAME_MAX - 10 + ud.mirrortarball = bb.fetch2.truncate_to_name_max(d.getVar("DL_DIR", True), ud.mirrortarball) ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball) + gitdir = d.getVar("GITDIR", True) or (d.getVar("DL_DIR", True) + "/git2/") + # The length of the filename should be less than NAME_MAX - 10 + gitsrcname = bb.fetch2.truncate_to_name_max(gitdir, gitsrcname) ud.clonedir = os.path.join(gitdir, gitsrcname) ud.localfile = ud.clonedir diff --git a/bitbake/lib/bb/fetch2/hg.py b/bitbake/lib/bb/fetch2/hg.py index b1c8675..be9bb26 100644 --- a/bitbake/lib/bb/fetch2/hg.py +++ b/bitbake/lib/bb/fetch2/hg.py @@ -66,6 +66,9 @@ class Hg(FetchMethod): ud.localfile = data.expand('%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision), d) + # The length of the filename should be less than NAME_MAX - 10 + ud.localfile = bb.fetch2.truncate_to_name_max(ud.pkgdir, ud.localfile) + def need_update(self, url, ud, d): revTag = ud.parm.get('rev', 'tip') if revTag == "tip": diff --git a/bitbake/lib/bb/fetch2/osc.py b/bitbake/lib/bb/fetch2/osc.py index 1a3a7bb..2e276e0 100644 --- a/bitbake/lib/bb/fetch2/osc.py +++ b/bitbake/lib/bb/fetch2/osc.py @@ -48,6 +48,8 @@ class Osc(FetchMethod): ud.revision = "" ud.localfile = data.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.path.replace('/', '.'), ud.revision), d) + # The length of the filename should be less than NAME_MAX - 10 + ud.localfile = bb.fetch2.truncate_to_name_max(ud.pkgdir, ud.localfile) def _buildosccommand(self, ud, d, command): """ diff --git a/bitbake/lib/bb/fetch2/repo.py b/bitbake/lib/bb/fetch2/repo.py index 8300da8..1e46968 100644 --- a/bitbake/lib/bb/fetch2/repo.py +++ b/bitbake/lib/bb/fetch2/repo.py @@ -52,6 +52,8 @@ class Repo(FetchMethod): ud.manifest += '.xml' ud.localfile = data.expand("repo_%s%s_%s_%s.tar.gz" % (ud.host, ud.path.replace("/", "."), ud.manifest, ud.branch), d) + # The length of the filename should be less than NAME_MAX - 10 + ud.localfile = bb.fetch2.truncate_to_name_max(d.getVar("DL_DIR", True), ud.localfile) def download(self, loc, ud, d): """Fetch url""" @@ -62,6 +64,8 @@ class Repo(FetchMethod): gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", ".")) repodir = data.getVar("REPODIR", d, True) or os.path.join(data.getVar("DL_DIR", d, True), "repo") + # The length of the filename should be less than NAME_MAX - 10 + gitsrcname = bb.fetch2.truncate_to_name_max(repodir, gitsrcname) codir = os.path.join(repodir, gitsrcname, ud.manifest) if ud.user: diff --git a/bitbake/lib/bb/fetch2/svk.py b/bitbake/lib/bb/fetch2/svk.py index ee3823f..dbd78f3 100644 --- a/bitbake/lib/bb/fetch2/svk.py +++ b/bitbake/lib/bb/fetch2/svk.py @@ -53,6 +53,8 @@ class Svk(FetchMethod): ud.revision = ud.parm.get('rev', "") ud.localfile = data.expand('%s_%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision, ud.date), d) + # The length of the filename should be less than NAME_MAX - 10 + ud.localfile = bb.fetch2.truncate_to_name_max(d.getVar("DL_DIR", True), ud.localfile) def need_update(self, url, ud, d): if ud.date == "now": diff --git a/bitbake/lib/bb/fetch2/svn.py b/bitbake/lib/bb/fetch2/svn.py index 9a779d2..c155735 100644 --- a/bitbake/lib/bb/fetch2/svn.py +++ b/bitbake/lib/bb/fetch2/svn.py @@ -65,6 +65,9 @@ class Svn(FetchMethod): ud.localfile = data.expand('%s_%s_%s_%s_.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision), d) + # The length of the filename should be less than NAME_MAX - 10 + ud.localfile = bb.fetch2.truncate_to_name_max(ud.pkgdir, ud.localfile) + def _buildsvncommand(self, ud, d, command): """ Build up an svn commandline based on ud
I think we may need to do something which is more context aware to ensure we preserve the most critical pieces of the filename which may be at the start or end or elsewhere depending on the fetcher implementation.
I agree with Richard. Truncation is most likely removing the most valuable disambiguation part of the filename. ie. host and path are more likely to be used for multiple SRC_URIs. Additionally the various git.py, svn.py... have a better view of the string and thus can do a better job of handling how best to shrink the filename. For example for git it makes sense to keep the hostname and drop the first parts of the pathname (again they are more likely to be non-unique). Instead of just dropping characters should we instead use some sort of ellipses? Is there any opportunity to know that a PREMIRROR is being used which we can drop? Maybe we should just start to catching the overflow occurrences and providing a better error/warning message.
Bulk move of 64 bugs to 4.3 M3 after a quick review. If a bug is actually fixed, please add a commit link and resolve it. -- Randy for YP bug team.
Moved to M4.
Bulk move to 5.0 M1. -- Randy
This hasn't been fixed or discussed much since 2013, I've lowered the importance and set the milestone to future.