Hamza/sovereign_ai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Inventory local runtime artifacts in SBOM

Browse Source
This commit is contained in:
Hamza Ayed
2026-10-07 17:57:25 +03:00
parent a09e763ada
commit 20c90dbf29
6 changed files with 393 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
SovereignAI-Starter/scripts/generate_component_inventory.py
+180 -6
View File
@@ -6,16 +6,159 @@ import argparse
import gzip
import hashlib
import importlib.metadata
import importlib.util
import json
import os
import re
from datetime import UTC, datetime
from pathlib import Path
from urllib.parse import unquote, urlparse
from uuid import uuid4
EASYOCR_RUNTIME_MODELS = {
"arabic.pth": {
"url": "https://github.com/JaidedAI/EasyOCR/releases/download/pre-v1.1.6/arabic.zip",
"md5": "993074555550e4e06a6077d55ff0449a",
},
"english_g2.pth": {
"url": "https://github.com/JaidedAI/EasyOCR/releases/download/v1.3/english_g2.zip",
"md5": "5864788e1821be9e454ec108d61b887d",
},
"craft_mlt_25k.pth": {
"url": "https://github.com/JaidedAI/EasyOCR/releases/download/pre-v1.1.6/craft_mlt_25k.zip",
"md5": "2f8227d2def4037cdb3b34389dcf9ec1",
},
}
def file_hash(path: Path, algorithm: str = "sha256") -> str:
hasher = (
hashlib.md5(usedforsecurity=False)
if algorithm == "md5"
else hashlib.new(algorithm)
)
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
hasher.update(chunk)
return hasher.hexdigest()
def digest(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
return file_hash(path)
def file_component(
path: Path,
*,
name: str,
component_type: str = "file",
scope: str,
license_status: str,
evidence: dict[str, str] | None = None,
) -> dict[str, object]:
sha256 = digest(path)
properties = [
{"name": "inventory.scope", "value": scope},
{"name": "inventory.observed_filename", "value": path.name},
{"name": "inventory.size_bytes", "value": str(path.stat().st_size)},
{"name": "inventory.license_status", "value": license_status},
]
properties.extend(
{"name": f"inventory.evidence.{key}", "value": value}
for key, value in (evidence or {}).items()
)
return {
"type": component_type,
"bom-ref": f"file:{name}:sha256:{sha256}",
"name": name,
"hashes": [{"alg": "SHA-256", "content": sha256}],
"properties": properties,
}
def _ocr_model_directory() -> Path:
configured = os.getenv("LOCAL_OCR_MODEL_DIR", "").strip()
if configured:
return Path(configured).expanduser()
local_app_data = os.getenv("LOCALAPPDATA")
base = Path(local_app_data) if local_app_data else Path.home() / ".local" / "share"
return base / "SovereignAI" / "models" / "easyocr"
def ocr_runtime_components(model_dir: Path) -> tuple[list[dict[str, object]], list[str]]:
"""Record installed EasyOCR weights and report requested model files that are absent."""
components: list[dict[str, object]] = []
missing: list[str] = []
for filename, source in EASYOCR_RUNTIME_MODELS.items():
path = model_dir / filename
if not path.is_file():
missing.append(filename)
continue
md5 = file_hash(path, "md5") # upstream manifest uses MD5 for artifact identity only
components.append(
file_component(
path,
name=f"EasyOCR model {filename}",
component_type="machine-learning-model",
scope="local-runtime-artifact",
license_status="model redistribution terms not established; human review required",
evidence={
"upstream_manifest": "easyocr==1.7.2 config.py",
"upstream_url": source["url"],
"upstream_manifest_md5": source["md5"],
"observed_md5": md5,
"upstream_md5_match": str(md5 == source["md5"]).lower(),
},
)
)
return components, missing
def pdfium_runtime_components() -> list[dict[str, object]]:
"""Hash the installed PDFium native binary and its wheel-bundled notice evidence."""
try:
spec = importlib.util.find_spec("pypdfium2_raw")
except (ImportError, ValueError):
return []
if spec is None or not spec.submodule_search_locations:
return []
package_root = Path(next(iter(spec.submodule_search_locations)))
binaries = [package_root / name for name in ("pdfium.dll", "libpdfium.so", "libpdfium.dylib")]
components: list[dict[str, object]] = []
try:
distribution = importlib.metadata.distribution("pypdfium2")
except importlib.metadata.PackageNotFoundError:
distribution = None
notice_hashes: dict[str, str] = {}
if distribution is not None:
for item in distribution.files or ():
relative = str(item).replace("\\", "/")
if "/BUILD_LICENSES/" not in relative:
continue
notice = Path(distribution.locate_file(item))
if notice.is_file():
notice_hashes[relative] = digest(notice)
for binary in binaries:
if binary.is_file():
components.append(
file_component(
binary,
name=f"PDFium native binary {binary.name}",
scope="local-runtime-artifact",
license_status="upstream and bundled dependency notices recorded; release review required",
evidence={
"python_distribution": (
f"pypdfium2 {distribution.version}"
if distribution is not None
else "pypdfium2 distribution metadata unavailable"
),
"bundled_build_license_files": json.dumps(
notice_hashes, ensure_ascii=False, sort_keys=True
),
},
)
)
return components
def canonical_name(value: str) -> str:
@@ -238,7 +381,14 @@ def main() -> int:
args = parser.parse_args()
root = args.project_root.resolve()
output = args.output or root / "sbom" / "component-inventory.json"
components = python_components(root) + flutter_components(root)
ocr_components, missing_ocr_models = ocr_runtime_components(_ocr_model_directory())
pdfium_components = pdfium_runtime_components()
components = (
python_components(root)
+ flutter_components(root)
+ pdfium_components
+ ocr_components
)
metadata: dict[str, object] = {
"timestamp": datetime.now(UTC).isoformat(),
"tools": [{"name": "generate_component_inventory.py"}],
@@ -264,8 +414,16 @@ def main() -> int:
"name": "inventory.flutter_lock_sha256",
"value": digest(root / "flutter_app" / "pubspec.lock"),
},
{
"name": "inventory.runtime_artifact_scope",
"value": "Observed files from the current host only; not a release artifact manifest.",
},
],
}
metadata["properties"].extend(
{"name": "inventory.missing_ocr_model", "value": filename}
for filename in missing_ocr_models
)
if args.flutter_notices:
notice_bytes = args.flutter_notices.read_bytes()
metadata["properties"].extend(
@@ -292,9 +450,21 @@ def main() -> int:
output.write_text(
json.dumps(document, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
python_count = sum(item["purl"].startswith("pkg:pypi/") for item in components)
pub_count = sum(item["purl"].startswith("pkg:pub/") for item in components)
missing = sum("licenses" not in item for item in components)
python_count = sum(str(item.get("purl", "")).startswith("pkg:pypi/") for item in components)
pub_count = sum(str(item.get("purl", "")).startswith("pkg:pub/") for item in components)
missing_library_license_metadata = sum(
item.get("type") == "library" and "licenses" not in item
for item in components
)
runtime_license_review = sum(
item.get("type") in {"file", "machine-learning-model"}
and any(
prop["name"] == "inventory.license_status"
and "review required" in prop["value"]
for prop in item["properties"]
)
for item in components
)
unclassified = sum(
any(
prop["name"] == "inventory.license_source"
@@ -313,7 +483,11 @@ def main() -> int:
"output": str(output),
"python_components": python_count,
"flutter_components": pub_count,
"components_missing_license_evidence": missing,
"pdfium_runtime_artifacts": len(pdfium_components),
"ocr_runtime_artifacts": len(ocr_components),
"missing_ocr_models": missing_ocr_models,
"library_components_missing_license_metadata": missing_library_license_metadata,
"runtime_artifacts_requiring_license_review": runtime_license_review,
"license_files_hashed": hashed_licenses,
"license_files_requiring_manual_classification": unclassified,
"complete_commercial_audit": False,