diff --git a/README.md b/README.md index 392ef1c..e3dfdc2 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,9 @@ The repository is organized around a practical study loop: ## Current Study Snapshot +> **⚠️ Important Note on Protection Score:** +> The `Protection Score` is a **proxy metric** that measures drift in ResNet18 feature space. It is **NOT validated** against real-world mimicry attacks like DreamBooth or LoRA. This score is useful for relative comparisons within this repository only, not as a universal guarantee of protection effectiveness. Real-world effectiveness has not been empirically validated. + Current local report highlights: | Run | Protection Score | PSNR | SSIM | Notes | @@ -57,8 +60,6 @@ Current local report highlights: | `blindfold` | `61.1` | `26.53` | `0.6114` | strongest current anti-readability preset, largest fidelity cost | | `collective n000050 / set_B` | `22.8` avg | `37.78` avg | `0.9666` avg | correct benchmark direction, objective still needs tuning | -The `Protection Score` is an internal proxy derived from embedding and style similarity after robustness transforms. It is useful for relative comparisons inside this repository, not as a universal guarantee against all AI systems. - ## Workflow 1. **Prepare input**: choose a single artwork, a folder, or a subject split such as `set_B`. diff --git a/src/auralock/cli.py b/src/auralock/cli.py index 09cb93c..1763d99 100644 --- a/src/auralock/cli.py +++ b/src/auralock/cli.py @@ -97,6 +97,11 @@ def _render_readability_table(report: dict[str, object]) -> Table: f"{report['protection_score']:.1f}/100", ) table.add_row("Assessment", str(report["assessment"])) + + # Add warning row if present + if "warning" in report: + table.add_row("⚠️ Warning", str(report["warning"])) + return table @@ -433,6 +438,13 @@ def protect( console.print(_render_protection_table(result)) console.print(_render_quality_table(result.quality_report)) console.print(_render_readability_table(result.protection_report)) + + # Print prominent warning about protection score + if "warning" in result.protection_report: + console.print(f"\n[yellow]{result.protection_report['warning']}[/yellow]") + if "disclaimer" in result.protection_report: + console.print(f"[dim]{result.protection_report['disclaimer']}[/dim]\n") + if report is not None: payload = result.to_report_dict(output_path=output_path) if adaptive_requirements is not None: @@ -485,6 +497,17 @@ def analyze( console.print(_render_quality_table(analysis_report["quality_report"])) console.print(_render_readability_table(analysis_report["protection_report"])) + + # Print prominent warning about protection score + if "warning" in analysis_report["protection_report"]: + console.print( + f"\n[yellow]{analysis_report['protection_report']['warning']}[/yellow]" + ) + if "disclaimer" in analysis_report["protection_report"]: + console.print( + f"[dim]{analysis_report['protection_report']['disclaimer']}[/dim]\n" + ) + if report_path is not None: _write_json_report( report_path, @@ -578,6 +601,16 @@ def batch( progress.update(task, completed=True, description="Batch processing completed") console.print(_render_batch_table(summary)) + + # Print prominent warning about protection scores in batch + from auralock.core.metrics import ( + PROTECTION_SCORE_DISCLAIMER, + PROTECTION_SCORE_WARNING, + ) + + console.print(f"\n[yellow]{PROTECTION_SCORE_WARNING}[/yellow]") + console.print(f"[dim]{PROTECTION_SCORE_DISCLAIMER}[/dim]\n") + if report is not None: _write_json_report(report, summary.to_report_dict()) console.print(f"[green]Report saved to:[/green] {report}") @@ -646,6 +679,16 @@ def benchmark( progress.update(task, completed=True, description="Benchmark completed") console.print(_render_profile_summary_table(summary.profile_summaries)) + + # Print prominent warning about protection scores in benchmark summary + from auralock.core.metrics import ( + PROTECTION_SCORE_DISCLAIMER, + PROTECTION_SCORE_WARNING, + ) + + console.print(f"\n[yellow]{PROTECTION_SCORE_WARNING}[/yellow]") + console.print(f"[dim]{PROTECTION_SCORE_DISCLAIMER}[/dim]\n") + if report is not None: _write_json_report(report, summary.to_report_dict()) console.print(f"[green]Report saved to:[/green] {report}") diff --git a/src/auralock/core/metrics.py b/src/auralock/core/metrics.py index 93274d2..893c380 100644 --- a/src/auralock/core/metrics.py +++ b/src/auralock/core/metrics.py @@ -24,6 +24,18 @@ load_default_style_feature_extractor, ) +# Protection Score Warning Constants +PROTECTION_SCORE_WARNING = ( + "⚠️ Protection Score is a proxy metric. " + "Real-world effectiveness is NOT validated." +) +PROTECTION_SCORE_METRIC_TYPE = "proxy_unvalidated" +PROTECTION_SCORE_DISCLAIMER = ( + "This score measures drift in ResNet18 feature space, " + "NOT actual mimicry prevention against real-world attacks like DreamBooth or LoRA. " + "Use for relative comparisons within this repository only." +) + def calculate_psnr( original: torch.Tensor | np.ndarray, @@ -311,11 +323,11 @@ def get_protection_readability_report( ) if protection_score >= 45: - assessment = "Strong" + assessment = "Strong (proxy space)" elif protection_score >= 25: - assessment = "Moderate" + assessment = "Moderate (proxy space)" else: - assessment = "Weak" + assessment = "Weak (proxy space)" return { "style_similarity": style_similarity, @@ -324,6 +336,9 @@ def get_protection_readability_report( "robust_embedding_similarity": robust_embedding_similarity, "protection_score": protection_score, "assessment": assessment, + "metric_type": PROTECTION_SCORE_METRIC_TYPE, + "warning": PROTECTION_SCORE_WARNING, + "disclaimer": PROTECTION_SCORE_DISCLAIMER, "transform_style_similarities": transform_style_similarities, "transform_embedding_similarities": transform_embedding_similarities, } diff --git a/src/tests/test_metrics.py b/src/tests/test_metrics.py index bff7236..1401c99 100644 --- a/src/tests/test_metrics.py +++ b/src/tests/test_metrics.py @@ -131,3 +131,70 @@ def test_quality_poor(self): report = get_quality_report(img1, img2) assert report["overall_quality"] == "Poor" + + +class TestProtectionReadabilityReport: + """Tests for protection readability report with warning metadata.""" + + def test_protection_report_includes_warnings(self): + """Test that protection report includes warning metadata.""" + from auralock.core.metrics import ( + PROTECTION_SCORE_DISCLAIMER, + PROTECTION_SCORE_METRIC_TYPE, + PROTECTION_SCORE_WARNING, + get_protection_readability_report, + ) + + original = torch.rand(1, 3, 100, 100) + modified = original + torch.randn_like(original) * 0.1 + modified = torch.clamp(modified, 0, 1) + + report = get_protection_readability_report(original, modified) + + # Check that warning metadata is present + assert "warning" in report + assert "metric_type" in report + assert "disclaimer" in report + + # Check that values match constants + assert report["warning"] == PROTECTION_SCORE_WARNING + assert report["metric_type"] == PROTECTION_SCORE_METRIC_TYPE + assert report["disclaimer"] == PROTECTION_SCORE_DISCLAIMER + + def test_protection_report_assessment_includes_proxy_space(self): + """Test that assessment labels include '(proxy space)' suffix.""" + from auralock.core.metrics import get_protection_readability_report + + original = torch.rand(1, 3, 100, 100) + modified = original + torch.randn_like(original) * 0.1 + modified = torch.clamp(modified, 0, 1) + + report = get_protection_readability_report(original, modified) + + # Check that assessment includes proxy space indicator + assert "(proxy space)" in report["assessment"] + + def test_protection_report_has_all_expected_keys(self): + """Test that protection report has all expected keys including new warning fields.""" + from auralock.core.metrics import get_protection_readability_report + + original = torch.rand(1, 3, 100, 100) + modified = original + torch.randn_like(original) * 0.1 + modified = torch.clamp(modified, 0, 1) + + report = get_protection_readability_report(original, modified) + + # Original keys + assert "style_similarity" in report + assert "embedding_similarity" in report + assert "robust_style_similarity" in report + assert "robust_embedding_similarity" in report + assert "protection_score" in report + assert "assessment" in report + assert "transform_style_similarities" in report + assert "transform_embedding_similarities" in report + + # New warning keys + assert "metric_type" in report + assert "warning" in report + assert "disclaimer" in report