Skip to content

Commit 276eb8f

Browse files
jbrunton96jbrunton96
authored
Add pipelines page to portal (Stirling-Tools#6818)
# Description of Changes Connect pipelines page to the backend. Note that this is really half an implementation because the portal doesn't have access to the tools list and their settings, but I can't fix that without re-architecture work, which I'll do in another PR, then come back to finish this off in a new PR. <img width="786" height="579" alt="image" src="https://github.com/user-attachments/assets/d3f06110-a35d-4d48-a2f9-1edb900c5c35" /> <img width="1232" height="519" alt="image" src="https://github.com/user-attachments/assets/9f344648-ea45-498d-9e84-9558a3999838" />
1 parent e44da5c commit 276eb8f

37 files changed

Lines changed: 2358 additions & 2350 deletions

‎app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import java.io.IOException;
44
import java.util.ArrayList;
5+
import java.util.Comparator;
56
import java.util.LinkedHashMap;
67
import java.util.List;
78
import java.util.Map;
@@ -52,11 +53,15 @@
5253
import stirling.software.proprietary.policy.model.PolicyRun;
5354
import stirling.software.proprietary.policy.model.PolicyRunStatus;
5455
import stirling.software.proprietary.policy.model.PolicyRunView;
56+
import stirling.software.proprietary.policy.overview.PoliciesOverviewResponse;
57+
import stirling.software.proprietary.policy.overview.PolicyOverviewService;
5558
import stirling.software.proprietary.policy.progress.PolicyProgressListener;
5659
import stirling.software.proprietary.policy.source.SourceAccessGuard;
5760
import stirling.software.proprietary.policy.source.SourceStore;
5861
import stirling.software.proprietary.policy.store.PolicyStore;
62+
import stirling.software.proprietary.policy.trigger.PolicyTrigger;
5963
import stirling.software.proprietary.policy.trigger.PolicyTriggerManager;
64+
import stirling.software.proprietary.policy.trigger.TriggerInfo;
6065

6166
/**
6267
* Policy CRUD plus pipeline runs (stored or ad-hoc). Runs are async: returns a run id, poll {@code
@@ -80,6 +85,8 @@ public class PolicyController {
8085
private final PolicyAccessGuard policyAccessGuard;
8186
private final PolicyManagementAuthority policyManagementAuthority;
8287
private final PolicyTriggerManager policyTriggerManager;
88+
private final PolicyOverviewService policyOverviewService;
89+
private final List<PolicyTrigger> policyTriggers;
8390
private final ApplicationProperties applicationProperties;
8491
private final TempFileManager tempFileManager;
8592
private final JobOwnershipService jobOwnershipService;
@@ -287,6 +294,31 @@ public List<Policy> listPolicies() {
287294
return policyAccessGuard.visibleFrom(policyStore);
288295
}
289296

297+
@GetMapping("/overview")
298+
@Operation(
299+
summary = "Pipelines overview",
300+
description =
301+
"Returns the KPI strip plus one row per policy the caller's team owns, each with"
302+
+ " its referenced sources resolved to names, its pipeline steps, and a"
303+
+ " trigger/output summary. Backs the portal's all-pipelines surface.")
304+
public PoliciesOverviewResponse overview() {
305+
return policyOverviewService.overview();
306+
}
307+
308+
@GetMapping("/triggers")
309+
@Operation(
310+
summary = "List available triggers",
311+
description =
312+
"Lists each trigger kind with whether it needs a source and which source types"
313+
+ " it supports, so the UI can offer triggers and pair them with the"
314+
+ " right sources.")
315+
public List<TriggerInfo> triggers() {
316+
return policyTriggers.stream()
317+
.map(TriggerInfo::of)
318+
.sorted(Comparator.comparing(TriggerInfo::type))
319+
.toList();
320+
}
321+
290322
@GetMapping("/{policyId}")
291323
@Operation(summary = "Get a policy by id")
292324
public ResponseEntity<Policy> getPolicy(@PathVariable String policyId) {
@@ -337,6 +369,26 @@ public ResponseEntity<JobResponse<Void>> runStoredPolicy(
337369
return ResponseEntity.accepted().body(new JobResponse<>(true, runId, null));
338370
}
339371

372+
@PostMapping("/{policyId}/trigger")
373+
@Operation(
374+
summary = "Run a stored policy against its sources",
375+
description =
376+
"Pulls the policy's configured sources and runs the pipeline now, regardless of"
377+
+ " the enabled flag (which only gates automatic triggering). Returns"
378+
+ " the ids of the runs started; poll the run-status endpoint for each."
379+
+ " Empty when the sources yielded no work to do.")
380+
public ResponseEntity<List<String>> trigger(@PathVariable String policyId) {
381+
Policy policy =
382+
policyStore
383+
.get(policyId)
384+
.filter(policyAccessGuard::canAccess)
385+
.orElseThrow(
386+
() ->
387+
new ResponseStatusException(
388+
HttpStatus.NOT_FOUND, "No policy: " + policyId));
389+
return ResponseEntity.accepted().body(policyRunner.run(policy));
390+
}
391+
340392
private static void requireRunnable(PipelineDefinition definition) {
341393
if (definition.steps().isEmpty()) {
342394
throw new ResponseStatusException(

‎app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunner.java‎

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package stirling.software.proprietary.policy.engine;
22

33
import java.io.IOException;
4+
import java.util.ArrayList;
45
import java.util.List;
56
import java.util.function.Consumer;
67

@@ -41,14 +42,15 @@ public class PolicyRunner {
4142
* Trigger entry point. Pulls every referenced source; each yielded unit becomes its own run so
4243
* one failure does not affect the others. No sources means one run with no input (generator
4344
* pipeline). Missing or disabled sources are skipped so one broken reference does not stop the
44-
* rest.
45+
* rest. Returns the ids of the runs it started (empty when sources yielded no work), so a
46+
* manual trigger can report back which runs to follow.
4547
*/
46-
public void run(Policy policy) {
48+
public List<String> run(Policy policy) {
4749
List<String> sourceIds = policy.sourceIds();
4850
if (sourceIds.isEmpty()) {
49-
startRun(policy, PolicyInputs.of(List.of()), unused -> {});
50-
return;
51+
return List.of(startRun(policy, PolicyInputs.of(List.of()), unused -> {}));
5152
}
53+
List<String> runIds = new ArrayList<>();
5254
for (String sourceId : sourceIds) {
5355
Source source = sourceStore.get(sourceId).orElse(null);
5456
if (source == null) {
@@ -63,8 +65,9 @@ public void run(Policy policy) {
6365
policy.id());
6466
continue;
6567
}
66-
pullAndRun(policy, source.toInputSpec());
68+
runIds.addAll(pullAndRun(policy, source.toInputSpec()));
6769
}
70+
return runIds;
6871
}
6972

7073
/** Run a stored policy on caller-supplied files (e.g. manual upload), bypassing its sources. */
@@ -79,14 +82,14 @@ public PolicyRunHandle runAdHoc(
7982
return policyEngine.submit(definition, inputs, listener);
8083
}
8184

82-
private void pullAndRun(Policy policy, InputSpec spec) {
85+
private List<String> pullAndRun(Policy policy, InputSpec spec) {
8386
InputSource source = sourceFor(spec);
8487
if (source == null) {
8588
log.warn(
8689
"No input source for type '{}' (policy {}); skipping",
8790
spec.type(),
8891
policy.id());
89-
return;
92+
return List.of();
9093
}
9194
List<ResolvedInput> work;
9295
try {
@@ -97,19 +100,22 @@ private void pullAndRun(Policy policy, InputSpec spec) {
97100
spec.type(),
98101
policy.id(),
99102
e.getMessage());
100-
return;
103+
return List.of();
101104
}
105+
List<String> runIds = new ArrayList<>();
102106
for (ResolvedInput unit : work) {
103-
startRun(policy, unit.inputs(), unit.onComplete());
107+
runIds.add(startRun(policy, unit.inputs(), unit.onComplete()));
104108
}
109+
return runIds;
105110
}
106111

107-
private void startRun(Policy policy, PolicyInputs inputs, Consumer<Boolean> onComplete) {
112+
private String startRun(Policy policy, PolicyInputs inputs, Consumer<Boolean> onComplete) {
108113
log.info("Running policy {} ({})", policy.id(), policy.name());
109114
PolicyRunHandle handle =
110115
policyEngine.runPolicy(policy, inputs, PolicyProgressListener.NOOP);
111116
handle.completion()
112117
.whenComplete((run, throwable) -> onComplete.accept(succeeded(run, throwable)));
118+
return handle.runId();
113119
}
114120

115121
private static boolean succeeded(PolicyRun run, Throwable throwable) {

‎app/proprietary/src/main/java/stirling/software/proprietary/policy/overview/PoliciesOverviewResponse.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
package stirling.software.proprietary.policy.overview;
2+
3+
import java.util.List;
4+
5+
/** The Pipelines overview payload: a KPI strip plus one row per policy. */
6+
public record PoliciesOverviewResponse(List<PolicyKpi> kpis, List<PolicyView> pipelines) {}

‎app/proprietary/src/main/java/stirling/software/proprietary/policy/overview/PolicyKpi.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
package stirling.software.proprietary.policy.overview;
2+
3+
/** One headline figure in the Pipelines overview strip. */
4+
public record PolicyKpi(long value, String description) {}

‎app/proprietary/src/main/java/stirling/software/proprietary/policy/overview/PolicyOverviewService.java‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
package stirling.software.proprietary.policy.overview;
2+
3+
import java.util.Comparator;
4+
import java.util.HashMap;
5+
import java.util.List;
6+
import java.util.Map;
7+
8+
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
9+
import org.springframework.stereotype.Service;
10+
11+
import lombok.RequiredArgsConstructor;
12+
13+
import stirling.software.proprietary.policy.config.PolicyAccessGuard;
14+
import stirling.software.proprietary.policy.model.OutputSpec;
15+
import stirling.software.proprietary.policy.model.PipelineStep;
16+
import stirling.software.proprietary.policy.model.Policy;
17+
import stirling.software.proprietary.policy.model.TriggerConfig;
18+
import stirling.software.proprietary.policy.source.Source;
19+
import stirling.software.proprietary.policy.source.SourceAccessGuard;
20+
import stirling.software.proprietary.policy.source.SourceStore;
21+
import stirling.software.proprietary.policy.store.PolicyStore;
22+
23+
/**
24+
* Builds the Pipelines overview: every policy the caller's team owns, each annotated with its
25+
* referenced sources (resolved to display names), its pipeline steps, and a trigger/output summary.
26+
* Source names are resolved from the team's sources in memory rather than persisted on the policy,
27+
* so the view always reflects the live source set. This is the "all pipelines" admin surface; the
28+
* user-facing Policies page builds only a friendly subset of the same backend policies.
29+
*/
30+
@Service
31+
@RequiredArgsConstructor
32+
@ConditionalOnBooleanProperty(name = "policies.enabled")
33+
public class PolicyOverviewService {
34+
35+
private final PolicyStore policyStore;
36+
private final SourceStore sourceStore;
37+
private final PolicyAccessGuard policyAccessGuard;
38+
private final SourceAccessGuard sourceAccessGuard;
39+
40+
public PoliciesOverviewResponse overview() {
41+
List<Policy> policies = policyAccessGuard.visibleFrom(policyStore);
42+
Map<String, String> sourceNames = sourceNames();
43+
44+
List<PolicyView> views =
45+
policies.stream()
46+
.map(policy -> toView(policy, sourceNames))
47+
.sorted(
48+
Comparator.comparing(
49+
PolicyView::name, String.CASE_INSENSITIVE_ORDER))
50+
.toList();
51+
52+
return new PoliciesOverviewResponse(buildKpis(policies), views);
53+
}
54+
55+
/** Display names for every source the caller's team can see, keyed by source id. */
56+
private Map<String, String> sourceNames() {
57+
Map<String, String> names = new HashMap<>();
58+
for (Source source : sourceAccessGuard.visibleFrom(sourceStore)) {
59+
names.put(source.id(), source.name());
60+
}
61+
return names;
62+
}
63+
64+
private static PolicyView toView(Policy policy, Map<String, String> sourceNames) {
65+
List<PolicyView.SourceRef> sources =
66+
policy.sourceIds().stream()
67+
// An unresolved id (source deleted, or not visible) falls back to the id so
68+
// the row still renders rather than dropping the reference silently.
69+
.map(id -> new PolicyView.SourceRef(id, sourceNames.getOrDefault(id, id)))
70+
.toList();
71+
List<String> steps = policy.steps().stream().map(PipelineStep::operation).toList();
72+
return new PolicyView(
73+
policy.id(),
74+
policy.name(),
75+
policy.enabled(),
76+
policy.enabled() ? "active" : "paused",
77+
triggerSummary(policy.trigger()),
78+
sources,
79+
steps,
80+
outputSummary(policy.output()),
81+
policy.owner());
82+
}
83+
84+
/** A null trigger is a manual-only policy; otherwise the trigger's type keys the summary. */
85+
private static String triggerSummary(TriggerConfig trigger) {
86+
return trigger == null ? "manual" : trigger.type();
87+
}
88+
89+
private static String outputSummary(OutputSpec output) {
90+
return output == null ? "inline" : output.type();
91+
}
92+
93+
private static List<PolicyKpi> buildKpis(List<Policy> policies) {
94+
long total = policies.size();
95+
long active = policies.stream().filter(Policy::enabled).count();
96+
long paused = total - active;
97+
return List.of(
98+
new PolicyKpi(total, "pipelines"),
99+
new PolicyKpi(active, "running automatically"),
100+
new PolicyKpi(paused, "paused"));
101+
}
102+
}

‎app/proprietary/src/main/java/stirling/software/proprietary/policy/overview/PolicyView.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package stirling.software.proprietary.policy.overview;
2+
3+
import java.util.List;
4+
5+
/**
6+
* One row in the Pipelines overview: a stored policy shown for the admin portal, with its
7+
* referenced sources resolved to names and its pipeline summarised. The portal's "all pipelines"
8+
* surface lists every backend policy (the user-facing Policies page builds only a friendly subset
9+
* of these).
10+
*/
11+
public record PolicyView(
12+
String id,
13+
String name,
14+
boolean enabled,
15+
String status,
16+
String trigger,
17+
List<SourceRef> sources,
18+
List<String> steps,
19+
String output,
20+
String owner) {
21+
22+
/** A source a policy pulls documents from, resolved to its display name. */
23+
public record SourceRef(String id, String name) {}
24+
}

‎app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/FolderWatchTrigger.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import lombok.extern.slf4j.Slf4j;
2828

2929
import stirling.software.common.model.ApplicationProperties;
30+
import stirling.software.proprietary.policy.config.FolderAccessGuard;
3031
import stirling.software.proprietary.policy.engine.PolicyRunner;
3132
import stirling.software.proprietary.policy.input.InputSource;
3233
import stirling.software.proprietary.policy.model.InputSpec;
@@ -74,6 +75,16 @@ public String type() {
7475
return TYPE;
7576
}
7677

78+
@Override
79+
public boolean requiresSource() {
80+
return true;
81+
}
82+
83+
@Override
84+
public Set<String> supportedSourceTypes() {
85+
return Set.of(FolderAccessGuard.FOLDER_TYPE);
86+
}
87+
7788
@Override
7889
public void validate(Policy policy) {
7990
if (watchDirsOf(policy).isEmpty()) {

‎app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/PolicyTrigger.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package stirling.software.proprietary.policy.trigger;
22

3+
import java.util.Set;
4+
35
import stirling.software.proprietary.policy.model.Policy;
46

57
/**
@@ -11,6 +13,24 @@ public interface PolicyTrigger {
1113
/** Matches {@code TriggerConfig.type()}. */
1214
String type();
1315

16+
/**
17+
* Whether this trigger needs at least one compatible input source to function. A schedule fires
18+
* on the clock regardless of sources, so it is false; folder-watch derives the directories it
19+
* watches from the policy's sources, so it is true. Drives whether the UI offers the trigger.
20+
*/
21+
default boolean requiresSource() {
22+
return false;
23+
}
24+
25+
/**
26+
* The source {@code type()}s this trigger is compatible with (e.g. {@code "folder"}). Empty
27+
* means source-agnostic (no constraint). Lets the UI offer a trigger only when a compatible
28+
* source is selected, without hard-coding the relationship.
29+
*/
30+
default Set<String> supportedSourceTypes() {
31+
return Set.of();
32+
}
33+
1434
/**
1535
* Validate at save time so misconfiguration fails fast, not at fire time. Receives the whole
1636
* {@link Policy} so triggers that depend on the policy's sources (folder-watch) can check that.

‎app/proprietary/src/main/java/stirling/software/proprietary/policy/trigger/TriggerInfo.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
package stirling.software.proprietary.policy.trigger;
2+
3+
import java.util.List;
4+
5+
/**
6+
* Describes an available trigger for the admin UI: its {@code type} (matching {@code
7+
* TriggerConfig.type()}), whether it needs a compatible source, and which source types it works
8+
* with. Lets the UI list supported triggers and pair them with sources without hard-coding the set.
9+
*/
10+
public record TriggerInfo(String type, boolean requiresSource, List<String> supportedSourceTypes) {
11+
12+
public static TriggerInfo of(PolicyTrigger trigger) {
13+
return new TriggerInfo(
14+
trigger.type(),
15+
trigger.requiresSource(),
16+
List.copyOf(trigger.supportedSourceTypes()));
17+
}
18+
}

0 commit comments

Comments
 (0)