-
Notifications
You must be signed in to change notification settings - Fork 514
Expand file tree
/
Copy pathtest_identify.py
More file actions
1959 lines (1609 loc) · 88.7 KB
/
Copy pathtest_identify.py
File metadata and controls
1959 lines (1609 loc) · 88.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""Tests for the provenance identifier (identify.py).
Pure attribution logic is unit-tested directly; end-to-end verdicts assert
against the real committed C2PA / IPTC fixtures in data/fixtures/provenance/.
"""
from __future__ import annotations
import base64
import hashlib
import json
import subprocess
import sys
from dataclasses import asdict
from pathlib import Path
from unittest.mock import patch
import pytest
from remove_ai_watermarks._internal.c2pa import c2pa_info_from_manifest_store
from remove_ai_watermarks._internal.constants import (
C2PA_AI_VENDORS,
C2PA_CLAIM_GENERATOR_PLATFORMS,
C2PA_IDENTITY_AI_ORGS,
)
from remove_ai_watermarks.identify import (
ProvenanceEvidence,
ProvenanceReport,
_ai_tools_in,
_attribute_platform,
_integrity_clashes,
_issuers_in,
_tc260_manufacturer_of,
_vendor_of,
evidence_from_metadata_record,
extract_provenance_evidence,
has_invisible_target,
identify,
identify_from_evidence,
)
from remove_ai_watermarks.watermark_registry import GEMINI_SPARKLE_TRUST_CONF
# Where the lazy import inside identify._visible_sparkle resolves the detector.
_SPARKLE_TARGET = "remove_ai_watermarks.gemini_engine.detect_sparkle_confidence"
SAMPLES_DIR = Path(__file__).resolve().parent.parent / "data" / "fixtures" / "provenance"
class TestProvenanceEvidence:
def test_exact_ai_claim_generator_can_assert_ai_without_source_type(self, tmp_path: Path):
path = tmp_path / "firefly.png"
info = {
"has_c2pa": True,
"issuer": "Adobe",
"claim_generator": "Adobe_Firefly",
"ai_tool": "Firefly",
"c2pa_identity_ai": True,
"c2pa_validation_source": "reader",
"c2pa_validation_state": "Valid",
"c2pa_integrity": "valid",
"c2pa_signature": "valid",
"c2pa_signer_trust": "untrusted",
"c2pa_signer_validity": "valid",
"c2pa_validation_codes": ["assertion.dataHash.match", "claimSignature.validated"],
}
evidence = ProvenanceEvidence(
path=path,
c2pa_info=info,
ai_metadata={},
scan=b"jumb c2pa Adobe_Firefly",
iptc_ai_system=None,
aigc_label=None,
exif_generator=None,
xai_signature=False,
huggingface_job=None,
samsung_genai=None,
)
report = identify_from_evidence(evidence)
assert report.is_ai_generated is True
# Intact binding and signature. The signer is not anchored, which is a missing
# input here (no trust bundle ships), not a finding against the credential.
assert report.confidence == "high"
assert report.platform == "Adobe Firefly"
def test_revoked_signing_credential_is_disqualifying(self, tmp_path: Path):
"""A credential the issuer disowned cannot establish origin.
Revocation arrives on its own dimension, not as a binding or signature failure,
so a check that reads only those two returned an AI verdict off a dead cert with
an empty ``integrity_clashes`` -- quieter than a hash mismatch on the same file.
The evidence comes from :func:`c2pa_info_from_manifest_store`, not a hand-written
dict of what it is believed to emit, so the assertion follows the producer when
its contract changes.
"""
path = tmp_path / "revoked.png"
info = c2pa_info_from_manifest_store(
{
"active_manifest": "created",
"validation_results": {
"activeManifest": {
"success": [
{"code": "assertion.dataHash.match"},
{"code": "claimSignature.validated"},
],
"failure": [{"code": "signingCredential.ocsp.revoked"}],
}
},
"manifests": {
"created": {
"signature_info": {"issuer": "OpenAI"},
"assertions": [
{
"label": "c2pa.actions.v2",
"data": {
"actions": [
{
"action": "c2pa.created",
"digitalSourceType": "trainedAlgorithmicMedia",
}
]
},
}
],
}
},
}
)
assert info["c2pa_signer_validity"] == "invalid"
evidence = ProvenanceEvidence(
path=path,
c2pa_info=info,
ai_metadata={},
scan=b"jumb c2pa OpenAI trainedAlgorithmicMedia",
iptc_ai_system=None,
aigc_label=None,
exif_generator=None,
xai_signature=False,
huggingface_job=None,
samsung_genai=None,
)
report = identify_from_evidence(evidence)
assert report.is_ai_generated is None
assert report.platform is None
assert report.confidence == "none"
assert any("revoked" in clash for clash in report.integrity_clashes)
def test_anchored_signer_is_also_high_confidence(self, tmp_path: Path):
path = tmp_path / "validated.png"
info = {
"has_c2pa": True,
"issuer": "OpenAI",
"source_type": "trainedAlgorithmicMedia (AI-generated)",
"ai_source_kind": "generated",
"c2pa_validation_source": "reader",
"c2pa_validation_state": "Trusted",
"c2pa_integrity": "valid",
"c2pa_signature": "valid",
"c2pa_signer_trust": "trusted",
"c2pa_signer_validity": "valid",
"c2pa_validation_codes": [
"assertion.dataHash.match",
"claimSignature.validated",
"signingCredential.trusted",
],
}
evidence = ProvenanceEvidence(
path=path,
c2pa_info=info,
ai_metadata={},
scan=b"jumb c2pa OpenAI trainedAlgorithmicMedia",
iptc_ai_system=None,
aigc_label=None,
exif_generator=None,
xai_signature=False,
huggingface_job=None,
samsung_genai=None,
)
report = identify_from_evidence(evidence)
assert report.is_ai_generated is True
assert report.confidence == "high"
assert report.platform == "OpenAI (ChatGPT / GPT Image / DALL·E / Sora)"
assert not any("not anchored" in caveat for caveat in report.caveats)
def test_external_metadata_record_builds_equivalent_evidence(self, tmp_path: Path):
path = tmp_path / "external.jpg"
signature = "A" * 64
artist = "c8045292-06d2-4c7d-b4f0-4f93b94e4801"
record = {
"pil": {"info:parameters": "Steps: 20, Sampler: Euler"},
"exif": {
"0th": {
"ImageDescription": f"Signature: {signature}",
"Artist": artist,
}
},
}
evidence = evidence_from_metadata_record(record, path=path)
report = identify_from_evidence(evidence)
assert evidence.path == path
assert evidence.ai_metadata["parameters"] == "Steps: 20, Sampler: Euler"
assert evidence.xai_signature is True
assert report.is_ai_generated is True
assert {signal.name for signal in report.signals} >= {"gen_params", "xai_signature"}
def test_external_scanner_diagnostics_do_not_create_c2pa_evidence(self, tmp_path: Path):
path = tmp_path / "plain.jpg"
record = {
"c2pa_store": {"error": "ManifestNotFound: no JUMBF data found"},
"jpeg": {
"segments": [
{
"marker": "APP11",
"kind": "c2pa_or_jumbf",
"base64": "AAA=",
}
]
},
}
report = identify_from_evidence(evidence_from_metadata_record(record, path=path))
assert report.is_ai_generated is None
assert report.signals == []
assert report.watermarks == []
def test_external_scanner_raw_bytes_still_create_c2pa_evidence(self, tmp_path: Path):
path = tmp_path / "signed.jpg"
manifest = b"jumb c2pa OpenAI trainedAlgorithmicMedia"
record = {
"jpeg": {
"segments": [
{
"marker": "APP11",
"kind": "c2pa_or_jumbf",
"base64": base64.b64encode(manifest).decode(),
}
]
}
}
report = identify_from_evidence(evidence_from_metadata_record(record, path=path))
assert report.is_ai_generated is True
assert report.platform == "OpenAI (ChatGPT / GPT Image / DALL·E / Sora)"
assert [signal.name for signal in report.signals] == ["c2pa"]
def test_external_generator_bytes_are_normalized(self, tmp_path: Path):
evidence = evidence_from_metadata_record(
{"exif": {"0th": {"Software": b"NovelAI"}}},
path=tmp_path / "external.png",
)
assert evidence.exif_generator == "NovelAI"
@pytest.mark.parametrize(
"record",
[
{"name": "trainedAlgorithmicMedia.jpg"},
{"sha256": "jumb-c2pa-OpenAI-trainedAlgorithmicMedia"},
{"pil": {"trainedAlgorithmicMedia": "plain"}},
{"signals": {"provenance": {"is_ai_generated": True}}},
{"pixel": {"error": "trainedAlgorithmicMedia"}},
],
)
def test_external_diagnostics_and_arbitrary_keys_are_not_evidence(self, tmp_path: Path, record: dict):
report = identify_from_evidence(evidence_from_metadata_record(record, path=tmp_path / "plain.jpg"))
assert report.is_ai_generated is None
assert report.signals == []
def test_external_metadata_value_is_evidence(self, tmp_path: Path):
record = {
"exif": {
"0th": {
"ImageDescription": "digitalSourceType=trainedAlgorithmicMedia",
}
}
}
report = identify_from_evidence(evidence_from_metadata_record(record, path=tmp_path / "generated.jpg"))
assert report.is_ai_generated is True
assert report.ai_source_kind == "generated"
@pytest.mark.parametrize(
"filename",
[
"chatgpt-1.png",
"chatgpt-2.png",
"doubao-1.png",
"firefly-1.png",
"flux-1.jpg",
"flux-1.png",
"grok-1.jpg",
"mj-1.png",
],
)
def test_metadata_only_identify_matches_extracted_evidence(self, filename: str):
path = SAMPLES_DIR / filename
direct = identify(path, check_visible=False, check_invisible=False)
evidence = extract_provenance_evidence(path)
extracted = identify_from_evidence(evidence)
assert isinstance(evidence, ProvenanceEvidence)
assert extracted == direct
def test_identify_from_evidence_does_not_read_the_source(self, monkeypatch, tmp_path: Path):
path = tmp_path / "generated.jpg"
path.write_bytes(b"\xff\xd8\xff\xe1jumbc2paOpenAI DALL-E trainedAlgorithmicMedia\xff\xd9")
evidence = extract_provenance_evidence(path)
def fail_if_called(*args, **kwargs):
raise AssertionError("identify_from_evidence must not read the source file")
monkeypatch.setattr("remove_ai_watermarks.identify.extract_c2pa_info", fail_if_called)
monkeypatch.setattr("remove_ai_watermarks.identify.get_ai_metadata", fail_if_called)
monkeypatch.setattr("remove_ai_watermarks.identify.scan_head", fail_if_called)
monkeypatch.setattr("remove_ai_watermarks.identify.iptc_ai_system", fail_if_called)
monkeypatch.setattr("remove_ai_watermarks.identify.aigc_label", fail_if_called)
monkeypatch.setattr("remove_ai_watermarks.identify.exif_generator", fail_if_called)
monkeypatch.setattr("remove_ai_watermarks.identify.xai_signature", fail_if_called)
monkeypatch.setattr("remove_ai_watermarks.identify.huggingface_job", fail_if_called)
monkeypatch.setattr("remove_ai_watermarks.identify.samsung_genai", fail_if_called)
monkeypatch.setattr("builtins.open", fail_if_called)
monkeypatch.setattr(Path, "open", fail_if_called)
report = identify_from_evidence(evidence)
assert report.is_ai_generated is True
assert any(signal.name == "c2pa" for signal in report.signals)
# ── Pure attribution logic (no file IO) ─────────────────────────────
class TestAttributePlatform:
def test_openai(self):
assert "OpenAI" in (_attribute_platform(["OpenAI"]) or "")
def test_designer_wins_over_openai_backend(self):
# Microsoft Designer signs as "OpenAI, Microsoft"; name the product.
platform = _attribute_platform(["OpenAI", "Microsoft"])
assert platform
assert "Designer" in platform
def test_adobe(self):
assert _attribute_platform(["Adobe"]) == "Adobe Firefly"
def test_google(self):
assert "Google" in (_attribute_platform(["Google LLC"]) or "")
def test_truepic_is_signer_not_generator(self):
platform = _attribute_platform(["Truepic"])
assert platform
assert "signer" in platform.lower()
def test_microsoft_label_is_model_neutral(self):
# Bing now runs MAI-Image, not DALL-E; the label must not claim DALL-E.
platform = _attribute_platform(["Microsoft"])
assert platform
assert "DALL-E" not in platform
def test_stability(self):
platform = _attribute_platform(["Stability AI"])
assert platform
assert "Stability AI" in platform
def test_canva(self):
platform = _attribute_platform(["Canva"])
assert platform
assert "Canva" in platform
def test_byteplus_keeps_its_product_name(self):
# ByteDance's intl brand signs as "Byteplus Pte. Ltd."; the registry maps
# it to the ByteDance family (was mis-read as Adobe via an incidental
# "Adobe XMP" file string before the entry existed).
platform = _attribute_platform(["BytePlus (ByteDance)"])
assert platform == "BytePlus (ByteDance)"
def test_empty_is_none(self):
assert _attribute_platform([]) is None
class TestIssuersIn:
def test_finds_openai(self):
assert _issuers_in(b"...OpenAI...trainedAlgorithmicMedia") == ["OpenAI"]
def test_finds_multiple_sorted(self):
assert _issuers_in(b"Microsoft and OpenAI") == ["Microsoft", "OpenAI"]
def test_none_present(self):
assert _issuers_in(b"just some bytes") == []
class TestAiToolsIn:
def test_finds_generator(self):
assert _ai_tools_in(b"...claim_generator Imagen 3...") == ["Imagen"]
def test_none_present(self):
assert _ai_tools_in(b"a regular photo, no tools") == []
class TestIdentifyNonPng:
"""Non-PNG containers (JPEG/WebP/AVIF) carry C2PA where the caBX parser can't
reach; identify recovers issuer + generator via the binary scan. Synthetic
byte blobs mirror tests/test_metadata.py::TestSynthIDSourceNonPng.
"""
def _c2pa_jpeg(self, tmp_path: Path, blob: bytes) -> Path:
path = tmp_path / "img.jpg"
path.write_bytes(b"\xff\xd8\xff\xe1jumbc2pa" + blob + b"\xff\xd9")
return path
def test_google_imagen_jpeg(self, tmp_path: Path):
path = self._c2pa_jpeg(tmp_path, b"Google Imagen ... trainedAlgorithmicMedia")
r = identify(path, check_visible=False)
assert r.is_ai_generated is True
assert r.platform is not None
assert "Google" in r.platform
# Generator recovered from the non-PNG blob shows up in the c2pa signal.
c2pa_signal = next(s for s in r.signals if s.name == "c2pa")
assert "Imagen" in c2pa_signal.detail
def test_openai_jpeg_has_synthid(self, tmp_path: Path):
path = self._c2pa_jpeg(
tmp_path,
b"OpenAI DALL-E ... trainedAlgorithmicMedia ... c2pa.watermarked.unbound",
)
r = identify(path, check_visible=False)
assert any("SynthID" in w for w in r.watermarks)
def test_black_forest_labs_flux_attributed(self, tmp_path: Path):
path = self._c2pa_jpeg(tmp_path, b"Black Forest Labs API ... trainedAlgorithmicMedia")
r = identify(path, check_visible=False, check_invisible=False)
assert r.is_ai_generated is True
assert r.platform == "Black Forest Labs (FLUX)"
def test_bytedance_volcengine_attributed(self, tmp_path: Path):
path = self._c2pa_jpeg(tmp_path, b"certificate_center@volcengine.com ... trainedAlgorithmicMedia")
r = identify(path, check_visible=False, check_invisible=False)
assert r.is_ai_generated is True
assert r.platform == "ByteDance Volcano Engine"
def test_bytedance_chinese_legal_name_attributed(self, tmp_path: Path):
# Some Volcano Engine certs name the signer with the Chinese legal entity
# rather than the latin "volcengine"; the latin needle misses it, so the
# Chinese-name registry entry is what attributes real ByteDance output.
blob = "北京火山引擎科技有限公司".encode() + b" ... trainedAlgorithmicMedia"
path = self._c2pa_jpeg(tmp_path, blob)
r = identify(path, check_visible=False, check_invisible=False)
assert r.is_ai_generated is True
assert r.platform == "ByteDance Volcano Engine"
@pytest.mark.parametrize(
("claim_generator", "platform"),
[
("Higgsfield AI", "Higgsfield AI"),
("recraft.ai", "Recraft"),
("Topaz Labs Image API", "Topaz Labs"),
("TIKTOK AD Creative Toolbox", "TikTok Ad Creative Toolbox"),
],
)
def test_claim_generator_wins_over_upstream_issuer(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, claim_generator: str, platform: str
):
path = tmp_path / "generated.png"
from PIL import Image
Image.new("RGB", (32, 32)).save(path)
monkeypatch.setattr(
"remove_ai_watermarks.identify.extract_c2pa_info",
lambda _path: {
"has_c2pa": True,
"issuer": "OpenAI",
"claim_generator": claim_generator,
"source_type": "trainedAlgorithmicMedia (AI-generated)",
"ai_source_kind": "generated",
},
)
report = identify(path, check_visible=False, check_invisible=False)
assert report.is_ai_generated is True
assert report.platform == platform
assert claim_generator in next(s.detail for s in report.signals if s.name == "c2pa")
def test_real_recraft_c2pa_fixture(self):
report = identify(SAMPLES_DIR / "recraft-v3.webp", check_visible=False)
assert report.is_ai_generated is True
assert report.confidence == "high"
assert report.platform == "Recraft"
assert any(signal.name == "c2pa" and "recraft.ai" in signal.detail for signal in report.signals)
def test_real_krea_api_fixture_has_no_local_provenance(self):
report = identify(SAMPLES_DIR / "krea-2-medium-turbo.png", check_visible=False)
assert report.is_ai_generated is None
assert report.platform is None
assert report.signals == []
def test_real_direct_kling_fixture_has_tc260_and_current_visible_mark(self):
path = SAMPLES_DIR.parent / "visible" / "kling" / "provider-original-direct.png"
report = identify(path, check_visible=True)
assert report.is_ai_generated is True
assert any(
signal.name == "aigc" and "001191110108335469089C10100" in signal.detail for signal in report.signals
)
assert any(signal.name == "visible_kling" for signal in report.signals)
@pytest.mark.parametrize("filename", ["qwen-image.png", "seedream-v4.jpg"])
def test_real_wavespeed_api_fixture_has_no_local_provenance(self, filename):
report = identify(SAMPLES_DIR / filename, check_visible=False)
assert report.is_ai_generated is None
assert report.platform is None
assert report.signals == []
def test_real_wavespeed_hunyuan_fixture_has_fal_c2pa(self):
path = SAMPLES_DIR / "hunyuan-image-3.png"
assert hashlib.sha256(path.read_bytes()).hexdigest() == (
"6ae2ac073ac79b02b0934c905364f200420c2ff60fea232bd9e0a5fca785cc09"
)
report = identify(path, check_visible=False, check_invisible=False)
assert report.is_ai_generated is True
assert report.platform == "fal.ai"
assert report.confidence == "high"
assert any(signal.name == "c2pa" and "fal-ai/hunyuan-image" in signal.detail for signal in report.signals)
def test_real_wavespeed_kling_fixture_has_tc260_metadata(self):
path = SAMPLES_DIR / "kling-image-v3.png"
assert hashlib.sha256(path.read_bytes()).hexdigest() == (
"501ab865b47ba59b64bc0f68118300744b0d5c7be2fa4e2ecbb3a78de5d8a82b"
)
report = identify(path, check_visible=False, check_invisible=False)
assert report.is_ai_generated is True
assert report.confidence == "high"
aigc = next(signal for signal in report.signals if signal.name == "aigc")
assert "001191110108335469089C10100" in aigc.detail
def test_real_qwen_create_fixture_has_tc260_metadata(self):
path = SAMPLES_DIR / "qwen-create-qwen-image-2.png"
assert hashlib.sha256(path.read_bytes()).hexdigest() == (
"60c39ed8aedb081193c62289be549785e74d31fc8d071961003fabd74712429d"
)
report = identify(path, check_visible=False, check_invisible=False)
assert report.is_ai_generated is True
assert report.confidence == "high"
aigc = next(signal for signal in report.signals if signal.name == "aigc")
assert "001191440101MA9Y9T4H7A00000" in aigc.detail
def test_dreamina_attributed_without_source_type(self, tmp_path: Path):
# Dreamina (ByteDance's international Jimeng brand) signs C2PA as
# "Bytedance Pte. Ltd." with a "Dreamina/x.y" claim generator and NO
# digitalSourceType assertion -- the generator name is the only AI signal.
# It is an identity-AI vendor (a pure generator), so attribution must not
# depend on trainedAlgorithmicMedia the way the incidental-mention-prone
# common-word issuers (Adobe/Google/OpenAI) do.
path = self._c2pa_jpeg(tmp_path, b"Bytedance Pte. Ltd. Dreamina/7.5.0 c2pa.created")
r = identify(path, check_visible=False, check_invisible=False)
assert r.is_ai_generated is True
assert r.platform == "ByteDance Dreamina"
def test_elevenlabs_attributed(self, tmp_path: Path):
path = self._c2pa_jpeg(tmp_path, b"Eleven Labs Inc. ... trainedAlgorithmicMedia")
r = identify(path, check_visible=False, check_invisible=False)
assert r.is_ai_generated is True
assert r.platform == "ElevenLabs"
assert not any("SynthID" in w for w in r.watermarks) # ElevenLabs does not use SynthID
def test_fal_ai_attributed(self, tmp_path: Path):
# fal.ai signs as "fal - Features & Labels Inc." with a "fal-ai/<model>"
# claim generator.
path = self._c2pa_jpeg(tmp_path, b"fal - Features & Labels Inc. fal-ai/seedvr trainedAlgorithmicMedia")
r = identify(path, check_visible=False, check_invisible=False)
assert r.is_ai_generated is True
assert r.platform == "fal.ai"
def test_bria_attributed_without_source_type(self, tmp_path: Path):
# Bria signs as "Bria Artificial Intelligence" with source type ``empty``
# (NO trainedAlgorithmicMedia) -- a pure-generator asserts_ai vendor, so
# the issuer/generator strings alone must flag AI.
path = self._c2pa_jpeg(tmp_path, b"Bria Artificial Intelligence Bria Ai c2pa.created c2pa.edited")
r = identify(path, check_visible=False, check_invisible=False)
assert r.is_ai_generated is True
assert r.platform == "Bria AI"
def test_stability_ai_issuer_attributed_no_synthid(self, tmp_path: Path):
path = self._c2pa_jpeg(tmp_path, b"Stability AI ... trainedAlgorithmicMedia")
r = identify(path, check_visible=False)
assert r.is_ai_generated is True
assert r.platform is not None
assert "Stability AI" in r.platform
assert not any("SynthID" in w for w in r.watermarks) # Stability does not use SynthID
def test_trained_source_is_generated_kind(self, tmp_path: Path):
path = self._c2pa_jpeg(tmp_path, b"OpenAI ... trainedAlgorithmicMedia")
r = identify(path, check_visible=False, check_invisible=False)
assert r.is_ai_generated is True
assert r.ai_source_kind == "generated"
def test_composite_source_is_enhanced_kind(self, tmp_path: Path):
# compositeWithTrainedAlgorithmicMedia: a real photo with an AI-composited
# region. Still AI (is_ai True), but the kind must read "enhanced" so a
# caller can do region-targeted cleaning instead of a full-frame regen.
path = self._c2pa_jpeg(tmp_path, b"Adobe ... compositeWithTrainedAlgorithmicMedia")
r = identify(path, check_visible=False, check_invisible=False)
assert r.is_ai_generated is True
assert r.ai_source_kind == "enhanced"
def test_c2pa_without_ai_marker_is_unknown(self, tmp_path: Path):
# Adobe signs C2PA on plain Photoshop edits too. Without an AI digital-
# source marker, the honest verdict is unknown -- the C2PA watermark is
# still listed, but is_ai_generated is not asserted True.
path = self._c2pa_jpeg(tmp_path, b"Adobe ... no ai marker here")
r = identify(path, check_visible=False)
assert r.is_ai_generated is None
assert any("C2PA" in w for w in r.watermarks)
assert not any("SynthID" in w for w in r.watermarks)
class TestIdentifySamsungGalaxy:
"""Samsung Galaxy / ASUS Gallery C2PA signers (verified on real signed files
2026-05-29; synthetic byte blobs here since the originals are private).
Galaxy AI edits stamp BOTH the device cert AND an AI source-type / genAIType,
so the signer attribution must NOT trip the camera-vs-AI integrity clash.
"""
def _jpeg(self, tmp_path: Path, name: str, blob: bytes) -> Path:
path = tmp_path / name
path.write_bytes(b"\xff\xd8\xff\xe1jumbc2pa" + blob + b"\xff\xd9")
return path
def test_galaxy_trained_source_is_unverified_ai(self, tmp_path: Path):
path = self._jpeg(tmp_path, "s25.jpg", b"Samsung Galaxy Galaxy S25 c2pa-rs trainedAlgorithmicMedia")
r = identify(path, check_visible=False, check_invisible=False)
assert r.is_ai_generated is True
assert r.confidence == "medium"
assert r.platform == "Samsung Galaxy (C2PA)"
assert r.c2pa_validation is None
assert any("without cryptographic validation" in caveat for caveat in r.caveats)
assert r.integrity_clashes == [] # device cert + AI source-type is legitimate, not a clash
def test_galaxy_genai_only_is_medium_ai(self, tmp_path: Path):
# The Galaxy S24 case: no trainedAlgorithmicMedia, genAIType is the only
# AI marker -- previously missed, now a medium-confidence verdict.
path = self._jpeg(
tmp_path, "s24.jpg", b'Samsung Galaxy Galaxy S24 c2pa-rs PhotoEditor_Re_Edit_Data{"genAIType":1}'
)
r = identify(path, check_visible=False, check_invisible=False)
assert r.is_ai_generated is True
assert r.confidence == "medium"
assert r.platform == "Samsung Galaxy (C2PA)"
assert any(s.name == "samsung_genai" for s in r.signals)
assert r.integrity_clashes == []
def test_asus_gallery_signer_not_ai(self, tmp_path: Path):
# ASUS Gallery signs edited photos; no AI source-type or genAIType, so the
# platform is attributed but the verdict stays unknown.
path = self._jpeg(tmp_path, "asus.jpg", b"/com.asus.gallery/3.8.0.98 c2pa-rs no ai marker")
r = identify(path, check_visible=False, check_invisible=False)
assert r.is_ai_generated is None
assert r.platform == "ASUS Gallery (C2PA signer)"
assert any("C2PA" in w for w in r.watermarks)
def test_galaxy_capture_without_ai_marker_is_not_ai(self, tmp_path: Path):
# A genuine Galaxy phone capture carries Samsung Galaxy C2PA provenance but
# NO AI source-type / genAIType. It must stay is_ai=None -- the device cert
# is authenticity provenance of a real photo, not an AI-generation signal.
path = self._jpeg(tmp_path, "s25_capture.jpg", b"Samsung Galaxy Galaxy S25 c2pa-rs no ai marker")
r = identify(path, check_visible=False, check_invisible=False)
assert r.is_ai_generated is None
assert r.platform == "Samsung Galaxy (C2PA)"
assert any("C2PA" in w for w in r.watermarks)
# ── End-to-end verdicts on real fixtures ────────────────────────────
@pytest.mark.skipif(not SAMPLES_DIR.exists(), reason="data/fixtures/provenance not present")
class TestIdentifyRealSamples:
def test_openai_chatgpt(self):
r = identify(SAMPLES_DIR / "chatgpt-1.png", check_visible=False)
assert r.is_ai_generated is True
assert r.confidence == "high"
assert r.platform
assert "OpenAI" in r.platform
assert any("C2PA" in w for w in r.watermarks)
assert not any("SynthID" in w for w in r.watermarks)
def test_adobe_firefly_has_no_synthid(self):
r = identify(SAMPLES_DIR / "firefly-1.png", check_visible=False)
assert r.is_ai_generated is True
assert r.platform == "Adobe Firefly"
assert not any("SynthID" in w for w in r.watermarks)
def test_iptc_made_with_ai(self):
# mj-1.png carries the IPTC digitalSourceType "Made with AI" marker.
r = identify(SAMPLES_DIR / "mj-1.png", check_visible=False)
assert r.is_ai_generated is True
assert any("IPTC" in w for w in r.watermarks)
def test_apple_clean_up_attributed(self, tmp_path: Path):
# Apple Photos Clean Up (Apple Intelligence object removal) marks the
# AI edit via photoshop:Credit next to compositeWithTrainedAlgorithmicMedia
# -- it must be attributed, not reported as a generic made-with-AI tag.
# This attribution must survive metadata consolidation.
p = tmp_path / "apple_cleanup.jpg"
p.write_bytes(
b'\xff\xd8\xff\xe1<x:xmpmeta photoshop:Credit="Apple Photos Clean Up" '
b"Iptc4xmpExt:DigitalSourceType=compositeWithTrainedAlgorithmicMedia></x:xmpmeta>\xff\xd9"
)
r = identify(p, check_visible=False, check_invisible=False)
assert r.is_ai_generated is True
assert r.platform == "Apple Photos (Clean Up AI edit)"
assert r.ai_source_kind == "enhanced"
assert "content_seal" not in [signal.name for signal in r.signals]
assert not any("Meta" in watermark for watermark in r.watermarks)
def test_standalone_iptc_composite_synthetic_is_enhanced(self, tmp_path: Path):
p = tmp_path / "composite.jpg"
p.write_bytes(
b'\xff\xd8\xff\xe1<x:xmpmeta Iptc4xmpExt:DigitalSourceType="compositeSynthetic"></x:xmpmeta>\xff\xd9'
)
r = identify(p, check_visible=False, check_invisible=False)
assert r.is_ai_generated is True
assert r.ai_source_kind == "enhanced"
def test_standalone_ai_tag_does_not_claim_a_vendor_watermark(self, tmp_path: Path):
"""The shared IPTC standard proves neither Meta nor Content Seal."""
p = tmp_path / "muse-tag.jpg"
p.write_bytes(
b'\xff\xd8\xff\xe1<x:xmpmeta Iptc4xmpExt:DigitalSourceType="trainedAlgorithmicMedia"></x:xmpmeta>\xff\xd9'
)
r = identify(p, check_visible=False, check_invisible=False)
names = [s.name for s in r.signals]
assert "iptc" in names
assert "content_seal" not in names
assert r.platform is None
assert not any("Content Seal" in watermark for watermark in r.watermarks)
def test_flux_bfl_c2pa_png(self):
# flux-1.png: real Black Forest Labs FLUX.2 Playground output (signed C2PA).
r = identify(SAMPLES_DIR / "flux-1.png", check_visible=False)
assert r.is_ai_generated is True
assert r.platform == "Black Forest Labs (FLUX)"
def test_flux_bfl_c2pa_jpeg_via_reader(self):
# flux-1.jpg: same source as a JPEG -- the real committed JPEG-with-C2PA
# fixture that exercises the c2pa-python non-PNG reader path end to end.
r = identify(SAMPLES_DIR / "flux-1.jpg", check_visible=False)
assert r.is_ai_generated is True
assert r.platform == "Black Forest Labs (FLUX)"
def test_clean_photo_is_unknown_not_clean(self, clean_photo: Path):
r = identify(clean_photo, check_visible=False)
assert r.is_ai_generated is None # never asserted False
assert r.platform is None
assert r.confidence == "none"
assert r.watermarks == []
def test_has_invisible_target_true_on_metadata_ai(self):
# The scrub gate: a C2PA/SynthID image and an IPTC "Made with AI" image are
# both invisible/metadata targets, so the diffusion scrub should run.
assert has_invisible_target(SAMPLES_DIR / "chatgpt-1.png") is True
assert has_invisible_target(SAMPLES_DIR / "mj-1.png") is True
# ai_from_metadata records scrub intent independently of the confidence string.
assert identify(SAMPLES_DIR / "chatgpt-1.png", check_visible=False).ai_from_metadata is True
def test_untrusted_but_intact_c2pa_is_high_confidence_with_a_caveat(self):
report = identify(SAMPLES_DIR / "chatgpt-1.png", check_visible=False, check_invisible=False)
assert report.is_ai_generated is True
assert report.confidence == "high"
assert report.ai_from_metadata is True
assert report.c2pa_validation is not None
assert report.c2pa_validation["source"] == "reader"
assert report.c2pa_validation["state"] == "Invalid"
assert report.c2pa_validation["integrity"] == "valid"
assert report.c2pa_validation["signature"] == "valid"
assert report.c2pa_validation["signer_trust"] == "untrusted"
assert report.c2pa_validation["signer_validity"] == "expired"
assert "assertion.dataHash.match" in report.c2pa_validation["codes"]
# What was not established is said, not folded into the confidence string.
assert any("never checked against one" in caveat for caveat in report.caveats)
assert any("only the signing time is unproven" in caveat for caveat in report.caveats)
def test_no_committed_fixture_reports_a_trusted_signer(self):
"""The reachability guard for :func:`_c2pa_credential_level`.
The SDK ships no production trust anchors, so ``signingCredential.trusted``
appears in no default installation. Gating high confidence on it made that branch
dead in production for every vendor while a hand-built dict kept it green in the
suite. These fixtures are the producer; if one ever comes back trusted, a bundle
got configured and the confidence mapping needs re-reading, not this assertion
deleted.
"""
checked = 0
for path in sorted(SAMPLES_DIR.iterdir()):
report = identify(path, check_visible=False, check_invisible=False)
if report.c2pa_validation is None:
continue
checked += 1
assert report.c2pa_validation["signer_trust"] != "trusted"
if report.c2pa_validation["integrity"] == "valid" and report.c2pa_validation["signature"] == "valid":
assert report.confidence == "high", path.name
assert checked >= 3
def test_hash_mismatch_does_not_confirm_origin_but_keeps_scrub_fail_safe(self, tampered_chatgpt_png: Path):
report = identify(tampered_chatgpt_png, check_visible=False, check_invisible=False)
assert report.is_ai_generated is None
assert report.platform is None
assert report.confidence == "none"
assert report.ai_source_kind is None
assert report.ai_from_metadata is False
assert report.c2pa_validation is not None
assert report.c2pa_validation["integrity"] == "invalid"
assert any("dataHash.mismatch" in clash for clash in report.integrity_clashes)
assert has_invisible_target(tampered_chatgpt_png) is True
def test_has_invisible_target_false_on_clean_photo(self, clean_photo: Path):
# No detectable invisible signal -> skip the scrub (do not degrade a clean image).
assert has_invisible_target(clean_photo) is False
assert identify(clean_photo, check_visible=False).ai_from_metadata is False
def test_strip_caveat_always_present(self, clean_photo: Path):
r = identify(clean_photo, check_visible=False)
assert any("not proof" in c for c in r.caveats)
def test_returns_report_dataclass(self):
assert isinstance(identify(SAMPLES_DIR / "firefly-1.png", check_visible=False), ProvenanceReport)
class TestHasInvisibleTargetFailSafe:
"""The scrub gate fails SAFE: when a detector errors, it runs the removal."""
def test_detector_error_defaults_to_run(self, tmp_path: Path):
# If evidence evaluation raises (a detector crash), the gate must return True so the
# caller still attempts removal -- leaving a watermark on a paid removal is
# worse than over-regenerating. (Garbage bytes do NOT raise; identify returns
# a clean None verdict there, so that path correctly skips -- see below.)
bad = tmp_path / "x.png"
bad.write_bytes(b"not image bytes")
with patch("remove_ai_watermarks.identify._identify_from_evidence", side_effect=RuntimeError("boom")):
assert has_invisible_target(bad) is True
def test_unreadable_bytes_are_not_a_target(self, tmp_path: Path):
# No raise, no signal -> not a scrub target (the CLI rejects undecodable
# images earlier anyway; this only documents the gate's own verdict).
bad = tmp_path / "x.png"
bad.write_bytes(b"not image bytes")
assert has_invisible_target(bad) is False
def test_local_ai_params_are_a_target(self, tmp_png_with_ai_metadata: Path):
assert has_invisible_target(tmp_png_with_ai_metadata) is True
# ── Local diffusion parameters (Stable Diffusion / ComfyUI) ─────────
class TestIdentifyLocalParams:
"""A PNG carrying SD-style generation params is attributed to a local pipeline."""
def test_sd_params_attributed_to_local_pipeline(self, tmp_png_with_ai_metadata: Path):
r = identify(tmp_png_with_ai_metadata, check_visible=False)
assert r.is_ai_generated is True
assert r.confidence == "high"
assert r.platform is not None
assert "Stable Diffusion" in r.platform
assert any("generation parameters" in w for w in r.watermarks)
def test_gen_params_signal_lists_keys(self, tmp_png_with_ai_metadata: Path):
r = identify(tmp_png_with_ai_metadata, check_visible=False)
signal = next(s for s in r.signals if s.name == "gen_params")
assert "parameters" in signal.detail
assert signal.confidence == "high"
def test_local_gen_params_have_no_c2pa_source_kind(self, tmp_png_with_ai_metadata: Path):
# AI verdict from local SD params (not C2PA) -> ai_source_kind stays None.
r = identify(tmp_png_with_ai_metadata, check_visible=False)
assert r.is_ai_generated is True
assert r.ai_source_kind is None
def test_clean_png_is_unknown(self, tmp_clean_png: Path):
r = identify(tmp_clean_png, check_visible=False)
assert r.is_ai_generated is None
assert r.platform is None
assert r.confidence == "none"
assert r.signals == []
# ── China TC260 AIGC label as a PNG text chunk (Doubao) ─────────────
class TestIdentifyAigcPngChunk:
"""The raw-JSON ``AIGC`` PNG chunk (no namespaced XMP marker) is a high-
confidence AI verdict, same as the XMP form."""
def _aigc_chunk_png(self, tmp_path: Path) -> Path:
from PIL import Image
from PIL.PngImagePlugin import PngInfo
p = tmp_path / "doubao_chunk.png"
pnginfo = PngInfo()
pnginfo.add_text("AIGC", json.dumps({"Label": "1", "ContentProducer": "doubao"}))
Image.new("RGB", (32, 32)).save(p, pnginfo=pnginfo)
return p
def test_png_chunk_detected_high(self, tmp_path: Path):
r = identify(self._aigc_chunk_png(tmp_path), check_visible=False)
assert r.is_ai_generated is True
assert r.confidence == "high"
assert r.platform is not None
assert "AIGC" in r.platform
signal = next(s for s in r.signals if s.name == "aigc")
assert "doubao" in signal.detail
# ── Hugging Face-hosted job marker (medium confidence) ─────────────
class TestIdentifyHuggingFaceJob:
"""The hf-job-id chunk lifts an otherwise-Unknown verdict to a tentative
(medium) AI, never overriding a high-confidence metadata signal."""
def _hf_png(self, tmp_path: Path) -> Path:
from PIL import Image
from PIL.PngImagePlugin import PngInfo
p = tmp_path / "hfjob.png"
pnginfo = PngInfo()
pnginfo.add_text("hf-job-id", "ec8380a6-2091-423a-b835-209420f99ee1")
Image.new("RGB", (32, 32)).save(p, pnginfo=pnginfo)
return p
def test_hf_job_promotes_to_medium(self, tmp_path: Path):
r = identify(self._hf_png(tmp_path), check_visible=False)
assert r.is_ai_generated is True
assert r.confidence == "medium"
assert r.platform is not None
assert "Hugging Face" in r.platform
signal = next(s for s in r.signals if s.name == "hf_job")
assert signal.confidence == "medium"
def test_hf_job_caveat_present(self, tmp_path: Path):
r = identify(self._hf_png(tmp_path), check_visible=False)
assert any("hf-job-id" in c for c in r.caveats)
def test_metadata_keeps_high_even_with_hf_job(self, tmp_png_with_ai_metadata: Path):
# A high-confidence metadata verdict is not downgraded by an hf-job hit.
from PIL import Image
from PIL.PngImagePlugin import PngInfo
img = Image.open(tmp_png_with_ai_metadata)
pnginfo = PngInfo()
for k, v in img.text.items():
pnginfo.add_text(k, v)
pnginfo.add_text("hf-job-id", "ec8380a6-2091-423a-b835-209420f99ee1")
img.save(tmp_png_with_ai_metadata, pnginfo=pnginfo)
r = identify(tmp_png_with_ai_metadata, check_visible=False)
assert r.confidence == "high"
# ── Visible-sparkle fallback (mocked detector) ──────────────────────
class TestIdentifyVisibleSparkle:
"""The visible-sparkle signal gates on the calibrated threshold (0.5)."""
def test_above_threshold_promotes_to_medium(self, tmp_clean_png: Path):
with patch(_SPARKLE_TARGET, return_value=0.7):
r = identify(tmp_clean_png, check_visible=True)
assert r.is_ai_generated is True
assert r.confidence == "medium"
assert r.platform is not None
assert "Gemini" in r.platform
signal = next(s for s in r.signals if s.name == "visible_sparkle")