[elbe-devel] [PATCH 5/8] elbepack: commands: parselicence: split up logic
Thomas Weißschuh
thomas.weissschuh at linutronix.de
Fri Aug 30 09:10:12 CEST 2024
From: Eduard Krein <eduard.krein at linutronix.de>
Split up logic to enhance readablility and further usage.
Move statistics into a seperate class and adapt the output.
Also adapt the parselicence test for the new output.
Signed-off-by: Eduard Krein <eduard.krein at linutronix.de>
---
elbepack/commands/parselicence.py | 135 ++++++++++--------
elbepack/tests/test_parselicence.py | 6 +-
.../tests/test_parselicence_reference.spdx | 4 +-
.../tests/test_parselicence_reference.xml | 34 ++---
4 files changed, 103 insertions(+), 76 deletions(-)
diff --git a/elbepack/commands/parselicence.py b/elbepack/commands/parselicence.py
index 0e9d61b6111d..4d4ed148b1f7 100644
--- a/elbepack/commands/parselicence.py
+++ b/elbepack/commands/parselicence.py
@@ -3,6 +3,7 @@
# SPDX-FileCopyrightText: 2016-2017 Linutronix GmbH
import argparse
+import dataclasses
import datetime
import io
import os
@@ -14,6 +15,17 @@ from elbepack.treeutils import etree
from elbepack.version import elbe_version
+ at dataclasses.dataclass
+class Statistics:
+ total_pkgs: int = 0
+ num_machine_readable: int = 0
+ num_heuristics: int = 0
+ num_error_pkgs: int = 0
+
+ def __str__(self):
+ return ' '.join([f'{k}={v}' for k, v in dataclasses.asdict(self).items()])
+
+
class license_dep5_to_spdx (dict):
def __init__(self, xml_fname=None):
super().__init__()
@@ -132,56 +144,13 @@ def license_string(pkg):
return ' AND '.join(l_list)
-def run_command(argv):
-
- aparser = argparse.ArgumentParser(prog='elbe parselicence')
- aparser.add_argument('--output', dest='output',
- help='outputfilename')
- aparser.add_argument('--mapping', dest='mapping',
- required=True,
- help='mapping filename')
- aparser.add_argument(
- '--use-nomos',
- action='store_true',
- dest='use_nomos',
- default=False,
- help='Use the external nomos tool on the copyright text, '
- 'and record the ouput in out xml')
- aparser.add_argument(
- '--errors-only',
- action='store_true',
- dest='only_errors',
- default=False,
- help='Only Output Packages with errors, '
- 'needing a fix in the mapping file')
- aparser.add_argument('--tvout', dest='tagvalue',
- help='tag value output filename')
-
- aparser.add_argument('licencefile')
-
- args = aparser.parse_args(argv)
-
- tree = etree(args.licencefile)
-
- num_pkg = 0
- mr = 0
- hr = 0
- err_pkg = 0
-
- mapping = license_dep5_to_spdx(args.mapping)
-
- # Dont use direct iterator, because we might want to delete
- # elements, when --errors-only is active
- for pkg in list(tree.root):
+def _apply_mapping(licenses, mapping, *, use_nomos=False, only_errors=False):
+ for pkg in list(licenses.root):
errors = []
pkg_name = pkg.et.attrib['name']
- num_pkg += 1
- if pkg.has('machinereadable'):
- mr += 1
if pkg.has('heuristics'):
- hr += 1
if not mapping.have_override(pkg_name):
errors.append(
f'no override for heuristics based package "{pkg_name}"')
@@ -220,7 +189,16 @@ def run_command(argv):
ll = sp.append('license')
ll.et.text = lic
- if args.use_nomos:
+ if errors:
+ for e in errors:
+ ee = pkg.append('error')
+ ee.et.text = e
+ elif only_errors:
+ # No Errors, and only_errors is active
+ # Remove package node
+ licenses.root.remove_child(pkg)
+
+ if use_nomos:
nomos_l = scan_nomos(pkg.text('text'))
if nomos_l[0] != 'No_license_found':
nomos_node = pkg.append('nomos_licenses')
@@ -229,15 +207,60 @@ def run_command(argv):
ll = nomos_node.append('license')
ll.et.text = lic
- if errors:
- for e in errors:
- ee = pkg.append('error')
- ee.et.text = e
- err_pkg += 1
- elif args.only_errors:
- # No Errors, and only_errors is active
- # Remove package node
- tree.root.remove_child(pkg)
+
+def _compute_statistics(licenses):
+ statistics = Statistics()
+ for pkg in list(licenses.root):
+
+ statistics.total_pkgs += 1
+
+ if pkg.has('machinereadable'):
+ statistics.num_machine_readable += 1
+
+ if pkg.has('heuristics'):
+ statistics.num_heuristics += 1
+
+ if pkg.has('error'):
+ statistics.num_error_pkgs += 1
+
+ return statistics
+
+
+def run_command(argv):
+
+ aparser = argparse.ArgumentParser(prog='elbe parselicence')
+ aparser.add_argument('--output', dest='output',
+ help='outputfilename')
+ aparser.add_argument('--mapping', dest='mapping',
+ required=True,
+ help='mapping filename')
+ aparser.add_argument(
+ '--use-nomos',
+ action='store_true',
+ dest='use_nomos',
+ default=False,
+ help='Use the external nomos tool on the copyright text, '
+ 'and record the ouput in out xml')
+ aparser.add_argument(
+ '--errors-only',
+ action='store_true',
+ dest='only_errors',
+ default=False,
+ help='Only Output Packages with errors, '
+ 'needing a fix in the mapping file')
+ aparser.add_argument('--tvout', dest='tagvalue',
+ help='tag value output filename')
+
+ aparser.add_argument('licencefile')
+
+ args = aparser.parse_args(argv)
+
+ tree = etree(args.licencefile)
+
+ mapping = license_dep5_to_spdx(args.mapping)
+
+ _apply_mapping(tree, mapping, use_nomos=args.use_nomos, only_errors=args.only_errors)
+ statistics = _compute_statistics(tree)
if args.tagvalue is not None:
created = datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='seconds')
@@ -273,4 +296,4 @@ def run_command(argv):
tree.write(args.output)
print('statistics:')
- print(f'num:{num_pkg} mr:{mr} hr:{hr} err_pkg:{err_pkg}')
+ print(statistics)
diff --git a/elbepack/tests/test_parselicence.py b/elbepack/tests/test_parselicence.py
index 3be19893c0d2..aaf30c08efe9 100644
--- a/elbepack/tests/test_parselicence.py
+++ b/elbepack/tests/test_parselicence.py
@@ -30,7 +30,11 @@ def test_parselicence(tmp_path):
here.joinpath('cyclonedx', 'build-simple-example', 'licence-target.xml'),
])
- assert stdout.getvalue() == 'statistics:\nnum:156 mr:137 hr:3 err_pkg:86\n'
+ assert stdout.getvalue() == 'statistics:'\
+ '\ntotal_pkgs=156' \
+ ' num_machine_readable=137' \
+ ' num_heuristics=3' \
+ ' num_error_pkgs=86\n'
xml_reference = here.joinpath('test_parselicence_reference.xml')
assert xml_output.read_text() == xml_reference.read_text()
diff --git a/elbepack/tests/test_parselicence_reference.spdx b/elbepack/tests/test_parselicence_reference.spdx
index 04236b6f571f..0daa5186d8fa 100644
--- a/elbepack/tests/test_parselicence_reference.spdx
+++ b/elbepack/tests/test_parselicence_reference.spdx
@@ -432,7 +432,7 @@ PackageLicenseInfoFromFiles: NOASSERTION
PackageName: libelf1
PackageDownloadLocation: NOASSERTION
PackageLicenseConcluded: NOASSERTION
-PackageLicenseDeclared: GPL-3.0-or-later AND UNKNOWN_MAPPING(GFDL-NIV-1.3) AND LGPL-2.1-or-later AND (LGPL-3.0-or-later OR GPL-2.0-or-later) AND GPL-3.0-or-later WITH Bison-exception-2.2 AND BSD-2-Clause
+PackageLicenseDeclared: UNKNOWN_MAPPING(GFDL-NIV-1.3) AND (LGPL-3.0-or-later OR GPL-2.0-or-later) AND GPL-3.0-or-later AND GPL-3.0-or-later WITH Bison-exception-2.2 AND BSD-2-Clause AND LGPL-2.1-or-later
PackageLicenseInfoFromFiles: NOASSERTION
## Package libext2fs2
@@ -481,7 +481,7 @@ PackageLicenseInfoFromFiles: NOASSERTION
PackageName: libgnutls30
PackageDownloadLocation: NOASSERTION
PackageLicenseConcluded: NOASSERTION
-PackageLicenseDeclared: Empty license AND BSD-3-Clause AND UNKNOWN_MAPPING(CC0 license) AND UNKNOWN_MAPPING(The main library is licensed under GNU Lesser) AND UNKNOWN_MAPPING(LGPLv3+_or_GPLv2+) AND MIT AND UNKNOWN_MAPPING(LGPLv2.1+) AND GPL-3.0-or-later AND Apache-2.0
+PackageLicenseDeclared: Empty license AND GPL-3.0-or-later AND Apache-2.0 AND UNKNOWN_MAPPING(The main library is licensed under GNU Lesser) AND UNKNOWN_MAPPING(CC0 license) AND UNKNOWN_MAPPING(LGPLv2.1+) AND UNKNOWN_MAPPING(LGPLv3+_or_GPLv2+) AND MIT AND BSD-3-Clause
PackageLicenseInfoFromFiles: NOASSERTION
## Package libgpg-error0
diff --git a/elbepack/tests/test_parselicence_reference.xml b/elbepack/tests/test_parselicence_reference.xml
index f712103c110d..cca0521b4660 100644
--- a/elbepack/tests/test_parselicence_reference.xml
+++ b/elbepack/tests/test_parselicence_reference.xml
@@ -17583,20 +17583,20 @@ Comment: GPL-3+
`/usr/share/common-licenses/GPL-3'
</text>
<heuristics/>
-<debian_licenses><license>GPL-3+</license>
-<license>GFDL-NIV-1.3</license>
-<license>LGPL-2.1+</license>
+<debian_licenses><license>GFDL-NIV-1.3</license>
<license>LGPL-3+ or GPL-2+</license>
+<license>GPL-3+</license>
<license>GPL-3+ with Bison exception</license>
<license>BSD-2-clause</license>
+<license>LGPL-2.1+</license>
</debian_licenses>
<spdx_licenses>
-<license>GPL-3.0-or-later</license>
<license>UNKNOWN_MAPPING(GFDL-NIV-1.3)</license>
-<license>LGPL-2.1-or-later</license>
<license>LGPL-3.0-or-later OR GPL-2.0-or-later</license>
+<license>GPL-3.0-or-later</license>
<license>GPL-3.0-or-later WITH Bison-exception-2.2</license>
<license>BSD-2-Clause</license>
+<license>LGPL-2.1-or-later</license>
</spdx_licenses><error>no override for heuristics based package "libelf1"</error>
<error>no mapping for "GFDL-NIV-1.3" for pkg "libelf1"</error>
</pkglicense>
@@ -22984,30 +22984,30 @@ Comment: https://github.com/benhoyt/inih
</text>
<heuristics/>
<debian_licenses><license/>
-<license>BSD-3-Clause</license>
-<license>CC0 license</license>
+<license>GPLv3+</license>
+<license>Apache-2.0</license>
<license>The main library is licensed under GNU Lesser</license>
+<license>CC0 license</license>
+<license>LGPLv2.1+</license>
<license>LGPLv3+_or_GPLv2+</license>
<license>Expat</license>
-<license>LGPLv2.1+</license>
-<license>GPLv3+</license>
-<license>Apache-2.0</license>
+<license>BSD-3-Clause</license>
</debian_licenses>
<spdx_licenses>
<license>Empty license</license>
-<license>BSD-3-Clause</license>
-<license>UNKNOWN_MAPPING(CC0 license)</license>
+<license>GPL-3.0-or-later</license>
+<license>Apache-2.0</license>
<license>UNKNOWN_MAPPING(The main library is licensed under GNU Lesser)</license>
+<license>UNKNOWN_MAPPING(CC0 license)</license>
+<license>UNKNOWN_MAPPING(LGPLv2.1+)</license>
<license>UNKNOWN_MAPPING(LGPLv3+_or_GPLv2+)</license>
<license>MIT</license>
-<license>UNKNOWN_MAPPING(LGPLv2.1+)</license>
-<license>GPL-3.0-or-later</license>
-<license>Apache-2.0</license>
+<license>BSD-3-Clause</license>
</spdx_licenses><error>no override for heuristics based package "libgnutls30"</error>
-<error>no mapping for "CC0 license" for pkg "libgnutls30"</error>
<error>no mapping for "The main library is licensed under GNU Lesser" for pkg "libgnutls30"</error>
-<error>no mapping for "LGPLv3+_or_GPLv2+" for pkg "libgnutls30"</error>
+<error>no mapping for "CC0 license" for pkg "libgnutls30"</error>
<error>no mapping for "LGPLv2.1+" for pkg "libgnutls30"</error>
+<error>no mapping for "LGPLv3+_or_GPLv2+" for pkg "libgnutls30"</error>
</pkglicense>
<pkglicense name="libgpg-error0"><text>Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
Upstream-Name: libgpg-error
--
2.46.0
More information about the elbe-devel
mailing list