Skip to content

Commit 398a517

Browse files
AAH20claude
andcommitted
Add AWS Config compliance-results-to-OSCAL-AR task
Adds a new 'aws-config-result-to-oscal-ar' task + transformer (mirroring the existing tanium/osco task+transformer shape) that converts AWS Config compliance evaluation results into OSCAL Assessment Results. Input: the json shape returned by AWS Config's get-compliance-details-by-config-rule / get-compliance-details-by-resource APIs (a top-level EvaluationResults list of EvaluationResult objects). Field names verified against: https://docs.aws.amazon.com/config/latest/APIReference/API_EvaluationResult.html https://docs.aws.amazon.com/config/latest/APIReference/API_EvaluationResultQualifier.html Output: one OSCAL Result per input file, with one Observation per EvaluationResult (method TEST-AUTOMATED), an inventory-item per distinct AWS resource (deduplicated across repeated evaluations of the same resource), and compliance-type/config-rule-name/etc. carried as props. Control mapping is intentionally include-all (AWS Config rules aren't mapped to specific catalog controls out of the box), same approach OSCO takes for the analogous case. Verified against a real installed trestle (editable install, Python 3.12): - 11 transformer-level tests (tests/trestle/transforms/implementations/ aws_config_test.py) against a schema-verified fixture -- inventory dedup, subject linkage, compliance-type/annotation props, missing optional fields, full oscal_serialize_json_bytes round-trip. - 6 task-level tests (tests/trestle/tasks/aws_config_result_to_oscal_ar_test.py) exercising the real TaskBase.execute()/simulate() path with actual file I/O, output-overwrite handling, and missing-config failure modes. - Confirmed 'trestle task -l' lists the new task correctly alongside osco-result-to-oscal-ar and tanium-result-to-oscal-ar. - flake8 clean on both new source files. Two real bugs were caught and fixed by this testing before submission: InventoryItem does not have a 'status' field (that's a SystemComponent field; my first draft wrongly copied it), and OSCAL's Observation.props requires min_length=1 when present, so it must be omitted rather than passed as an empty list. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent ff35838 commit 398a517

6 files changed

Lines changed: 694 additions & 0 deletions

File tree

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
{
2+
"EvaluationResults": [
3+
{
4+
"EvaluationResultIdentifier": {
5+
"EvaluationResultQualifier": {
6+
"ConfigRuleName": "s3-bucket-public-read-prohibited",
7+
"ResourceType": "AWS::S3::Bucket",
8+
"ResourceId": "acme-prod-uploads",
9+
"EvaluationMode": "DETECTIVE"
10+
},
11+
"OrderingTimestamp": "2026-08-16T09:12:03.000Z"
12+
},
13+
"ComplianceType": "NON_COMPLIANT",
14+
"ResultRecordedTime": "2026-08-16T09:12:05.000Z",
15+
"ConfigRuleInvokedTime": "2026-08-16T09:12:04.000Z",
16+
"Annotation": "The S3 bucket policy allows public read access.",
17+
"ResultToken": "tok-0001"
18+
},
19+
{
20+
"EvaluationResultIdentifier": {
21+
"EvaluationResultQualifier": {
22+
"ConfigRuleName": "s3-bucket-public-read-prohibited",
23+
"ResourceType": "AWS::S3::Bucket",
24+
"ResourceId": "acme-prod-assets",
25+
"EvaluationMode": "DETECTIVE"
26+
},
27+
"OrderingTimestamp": "2026-08-16T09:12:07.000Z"
28+
},
29+
"ComplianceType": "COMPLIANT",
30+
"ResultRecordedTime": "2026-08-16T09:12:08.000Z",
31+
"ConfigRuleInvokedTime": "2026-08-16T09:12:07.000Z",
32+
"ResultToken": "tok-0002"
33+
},
34+
{
35+
"EvaluationResultIdentifier": {
36+
"EvaluationResultQualifier": {
37+
"ConfigRuleName": "restricted-ssh",
38+
"ResourceType": "AWS::EC2::SecurityGroup",
39+
"ResourceId": "sg-0a1b2c3d4e5f6g7h8",
40+
"EvaluationMode": "DETECTIVE"
41+
},
42+
"OrderingTimestamp": "2026-08-16T09:13:11.000Z"
43+
},
44+
"ComplianceType": "NON_COMPLIANT",
45+
"ResultRecordedTime": "2026-08-16T09:13:12.000Z",
46+
"ConfigRuleInvokedTime": "2026-08-16T09:13:11.000Z",
47+
"Annotation": "Security group allows unrestricted SSH access (0.0.0.0/0 on port 22).",
48+
"ResultToken": "tok-0003"
49+
},
50+
{
51+
"EvaluationResultIdentifier": {
52+
"EvaluationResultQualifier": {
53+
"ConfigRuleName": "restricted-ssh",
54+
"ResourceType": "AWS::EC2::SecurityGroup",
55+
"ResourceId": "sg-0a1b2c3d4e5f6g7h8",
56+
"EvaluationMode": "DETECTIVE"
57+
},
58+
"OrderingTimestamp": "2026-08-15T09:13:11.000Z"
59+
},
60+
"ComplianceType": "NON_COMPLIANT",
61+
"ResultRecordedTime": "2026-08-15T09:13:12.000Z",
62+
"ConfigRuleInvokedTime": "2026-08-15T09:13:11.000Z",
63+
"Annotation": "Security group allows unrestricted SSH access (0.0.0.0/0 on port 22).",
64+
"ResultToken": "tok-0000"
65+
}
66+
]
67+
}
Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
# -*- mode:python; coding:utf-8 -*-
2+
# Copyright (c) 2026 IBM Corp. All rights reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# https://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
"""AWS Config to OSCAL task tests."""
16+
17+
import configparser
18+
import json
19+
import pathlib
20+
21+
from tests.test_utils import TEST_DIR
22+
23+
from trestle.tasks.aws_config_result_to_oscal_ar import AwsConfigResultToOscalAR
24+
from trestle.tasks.base_task import TaskOutcome
25+
26+
test_data_dir = TEST_DIR / 'data/tasks/aws-config'
27+
28+
29+
def _build_config(input_dir: pathlib.Path, output_dir: pathlib.Path) -> configparser.SectionProxy:
30+
config = configparser.ConfigParser()
31+
section = 'task.aws-config-result-to-oscal-ar'
32+
config.add_section(section)
33+
config.set(section, 'input-dir', str(input_dir))
34+
config.set(section, 'output-dir', str(output_dir))
35+
return config[section]
36+
37+
38+
class TestAwsConfigResultToOscalAR:
39+
def test_print_info(self, capsys):
40+
task = AwsConfigResultToOscalAR(None)
41+
task.print_info()
42+
43+
def test_missing_config_fails(self):
44+
task = AwsConfigResultToOscalAR(None)
45+
assert task.execute() == TaskOutcome.FAILURE
46+
47+
def test_missing_required_keys_fails(self):
48+
config = configparser.ConfigParser()
49+
config.add_section('task.aws-config-result-to-oscal-ar')
50+
task = AwsConfigResultToOscalAR(config['task.aws-config-result-to-oscal-ar'])
51+
assert task.execute() == TaskOutcome.FAILURE
52+
53+
def test_simulate_does_not_write_output(self, tmp_path):
54+
output_dir = tmp_path / 'out'
55+
config = _build_config(test_data_dir, output_dir)
56+
task = AwsConfigResultToOscalAR(config)
57+
outcome = task.simulate()
58+
assert outcome == TaskOutcome.SIM_SUCCESS
59+
assert not output_dir.exists() or not list(output_dir.iterdir())
60+
61+
def test_execute_produces_valid_oscal_result(self, tmp_path):
62+
output_dir = tmp_path / 'out'
63+
config = _build_config(test_data_dir, output_dir)
64+
task = AwsConfigResultToOscalAR(config)
65+
outcome = task.execute()
66+
assert outcome == TaskOutcome.SUCCESS
67+
68+
produced = list(output_dir.glob('*.oscal.json'))
69+
assert len(produced) == 1
70+
data = json.loads(produced[0].read_text(encoding='utf-8'))
71+
assert 'results' in data
72+
result = data['results'][0]
73+
assert len(result['observations']) == 4
74+
assert len(result['local-definitions']['inventory-items']) == 3
75+
76+
def test_execute_respects_output_overwrite_false(self, tmp_path):
77+
output_dir = tmp_path / 'out'
78+
config = _build_config(test_data_dir, output_dir)
79+
task = AwsConfigResultToOscalAR(config)
80+
assert task.execute() == TaskOutcome.SUCCESS
81+
82+
config2 = _build_config(test_data_dir, output_dir)
83+
config2['output-overwrite'] = 'false'
84+
task2 = AwsConfigResultToOscalAR(config2)
85+
assert task2.execute() == TaskOutcome.FAILURE
Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
# -*- mode:python; coding:utf-8 -*-
2+
# Copyright (c) 2026 IBM Corp. All rights reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# https://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
"""Tests for AwsConfigResultToOscalARTransformer.
16+
17+
The fixture (tests/data/tasks/aws-config/aws-config-sample.json) is
18+
hand-constructed to match AWS Config's documented EvaluationResult schema
19+
field-for-field (EvaluationResultIdentifier / EvaluationResultQualifier /
20+
ComplianceType / ResultRecordedTime / ConfigRuleInvokedTime / Annotation /
21+
ResultToken), verified against
22+
https://docs.aws.amazon.com/config/latest/APIReference/API_EvaluationResult.html
23+
and API_EvaluationResultQualifier.html -- it is not a captured real AWS API
24+
response (no AWS account was used), but every field name and shape is real.
25+
"""
26+
import pathlib
27+
28+
import pytest
29+
30+
from trestle.transforms.implementations.aws_config import AwsConfigResultToOscalARTransformer
31+
32+
test_data_dir = pathlib.Path('tests/data/tasks/aws-config').resolve()
33+
34+
35+
@pytest.fixture
36+
def sample_blob() -> str:
37+
return (test_data_dir / 'aws-config-sample.json').read_text(encoding='utf-8')
38+
39+
40+
class TestTransform:
41+
def test_produces_one_result_with_all_observations(self, sample_blob):
42+
transformer = AwsConfigResultToOscalARTransformer()
43+
results = transformer.transform(sample_blob)
44+
assert len(results.root) == 1
45+
result = results.root[0]
46+
# 4 EvaluationResults in the fixture -> 4 Observations (one per evaluation, even
47+
# when the same resource is evaluated more than once).
48+
assert len(result.observations) == 4
49+
50+
def test_inventory_deduplicated_by_resource(self, sample_blob):
51+
"""The same security group appears in 2 EvaluationResults -> 1 inventory item."""
52+
transformer = AwsConfigResultToOscalARTransformer()
53+
results = transformer.transform(sample_blob)
54+
result = results.root[0]
55+
inventory_items = result.local_definitions.inventory_items
56+
# 3 distinct resources: 2 S3 buckets + 1 security group (evaluated twice).
57+
assert len(inventory_items) == 3
58+
descriptions = {i.description for i in inventory_items}
59+
assert descriptions == {
60+
'AWS::S3::Bucket acme-prod-uploads',
61+
'AWS::S3::Bucket acme-prod-assets',
62+
'AWS::EC2::SecurityGroup sg-0a1b2c3d4e5f6g7h8',
63+
}
64+
65+
def test_observation_subjects_reference_the_correct_inventory_item(self, sample_blob):
66+
transformer = AwsConfigResultToOscalARTransformer()
67+
results = transformer.transform(sample_blob)
68+
result = results.root[0]
69+
sg_item_uuid = next(
70+
i.uuid for i in result.local_definitions.inventory_items
71+
if 'sg-0a1b2c3d4e5f6g7h8' in i.description
72+
)
73+
sg_observations = [o for o in result.observations if 'restricted-ssh' in o.description]
74+
assert len(sg_observations) == 2
75+
for obs in sg_observations:
76+
assert obs.subjects[0].subject_uuid == sg_item_uuid
77+
assert obs.subjects[0].type.root == 'inventory-item'
78+
79+
def test_compliance_type_and_annotation_carried_in_props(self, sample_blob):
80+
transformer = AwsConfigResultToOscalARTransformer()
81+
results = transformer.transform(sample_blob)
82+
result = results.root[0]
83+
non_compliant = [
84+
o for o in result.observations
85+
if any(p.name == 'compliance-type' and p.value == 'NON_COMPLIANT' for p in o.props)
86+
]
87+
# 3 of the 4 fixture entries are NON_COMPLIANT.
88+
assert len(non_compliant) == 3
89+
s3_obs = next(o for o in result.observations if 'acme-prod-uploads' in o.description)
90+
assert 'public read access' in s3_obs.description
91+
assert 'AWS::S3::Bucket' in s3_obs.description
92+
93+
def test_compliant_entry_has_no_annotation_but_valid_description(self, sample_blob):
94+
transformer = AwsConfigResultToOscalARTransformer()
95+
results = transformer.transform(sample_blob)
96+
result = results.root[0]
97+
compliant_obs = next(o for o in result.observations if 'acme-prod-assets' in o.description)
98+
assert compliant_obs.description == 's3-bucket-public-read-prohibited (AWS::S3::Bucket acme-prod-assets)'
99+
100+
def test_methods_and_collected_are_set(self, sample_blob):
101+
transformer = AwsConfigResultToOscalARTransformer()
102+
results = transformer.transform(sample_blob)
103+
result = results.root[0]
104+
for obs in result.observations:
105+
assert obs.methods == ['TEST-AUTOMATED']
106+
assert obs.collected is not None
107+
108+
def test_reviewed_controls_present(self, sample_blob):
109+
transformer = AwsConfigResultToOscalARTransformer()
110+
results = transformer.transform(sample_blob)
111+
result = results.root[0]
112+
assert result.reviewed_controls is not None
113+
assert result.reviewed_controls.control_selections is not None
114+
115+
def test_analysis_reports_correct_counts(self, sample_blob):
116+
transformer = AwsConfigResultToOscalARTransformer()
117+
results = transformer.transform(sample_blob)
118+
assert results.root # ensure transform ran before checking analysis
119+
assert 'inventory: 3' in transformer.analysis
120+
assert 'observations: 4' in transformer.analysis
121+
122+
def test_empty_evaluation_results_produces_result_with_no_observations(self):
123+
transformer = AwsConfigResultToOscalARTransformer()
124+
results = transformer.transform('{"EvaluationResults": []}')
125+
assert len(results.root) == 1
126+
assert results.root[0].observations is None
127+
128+
def test_missing_optional_fields_do_not_raise(self):
129+
"""A minimal, spec-legal EvaluationResult (all fields optional per AWS docs)."""
130+
blob = '{"EvaluationResults": [{}]}'
131+
transformer = AwsConfigResultToOscalARTransformer()
132+
results = transformer.transform(blob)
133+
result = results.root[0]
134+
assert len(result.observations) == 1
135+
assert result.observations[0].description == 'Unknown (Unknown Unknown)'
136+
137+
def test_result_round_trips_through_oscal_serialization(self, sample_blob):
138+
"""The Result must be genuinely OSCAL-schema-valid, not just pydantic-constructible."""
139+
transformer = AwsConfigResultToOscalARTransformer()
140+
results = transformer.transform(sample_blob)
141+
serialized = results.oscal_serialize_json_bytes(pretty=True)
142+
assert b'"aws-config-result"' not in serialized # sanity: not leaking internal names
143+
assert b'observations' in serialized
144+
assert b'restricted-ssh' in serialized

0 commit comments

Comments
 (0)