Add sources service and frontend (Stirling-Tools#6774)

# Description of Changes
Redesign policies backend to treat sources a lot closer to how the
frontend imagined them working (they're persistent now and have an API).
Then connect the portal to the sources when mocks are off to allow for
source creation in the UI. It's not particularly useful to do that right
now because there's no policies UI, but I've tested manually that
sources set up in the UI are usable by policies created via the API.

I had to change the portal so that when mocks are off, it doesn't just
hard crash when attempting to connect to all the backend APIs that don't
exist yet. It'll still log the errors, but just continues on rendering
the UI now.

I also changed all the policies backend APIs to be gated behind a flag
instead of behind the SaaS profile. This is because we haven't yet got
the payment model sorted, but we're going to need this stuff running
self-hosted to be able to test it locally.

Author: jbrunton96

.taskfiles/backend.yml

@@ -26,6 +26,7 @@ tasks:
           AIENGINE_ENABLED: '{{.AIENGINE_ENABLED}}'
           AIENGINE_TIMEOUTSECONDS: '{{.AIENGINE_TIMEOUTSECONDS}}'
           SECURITY_ENABLELOGIN: '{{.SECURITY_ENABLELOGIN}}'
+          POLICIES_ENABLED: '{{.POLICIES_ENABLED}}'
 
   dev:proprietary:
     desc: "Start backend dev server in proprietary mode"
@@ -36,12 +37,13 @@ tasks:
       AIENGINE_ENABLED: '{{.AIENGINE_ENABLED | default "false"}}'
       AIENGINE_TIMEOUTSECONDS: '{{.AIENGINE_TIMEOUTSECONDS | default "120"}}'
       SECURITY_ENABLELOGIN: '{{.SECURITY_ENABLELOGIN | default ""}}'
+      POLICIES_ENABLED: '{{.POLICIES_ENABLED | default ""}}'
     env:
       SERVER_PORT: '{{.PORT}}'
     cmds:
-      - cmd: '{{if .AIENGINE_URL}}AIENGINE_URL={{.AIENGINE_URL}} AIENGINE_ENABLED={{.AIENGINE_ENABLED}} AIENGINE_TIMEOUTSECONDS={{.AIENGINE_TIMEOUTSECONDS}} {{end}}{{if .SECURITY_ENABLELOGIN}}SECURITY_ENABLELOGIN={{.SECURITY_ENABLELOGIN}} {{end}}cmd /c ".\gradlew.bat :stirling-pdf:bootRun"'
+      - cmd: '{{if .AIENGINE_URL}}AIENGINE_URL={{.AIENGINE_URL}} AIENGINE_ENABLED={{.AIENGINE_ENABLED}} AIENGINE_TIMEOUTSECONDS={{.AIENGINE_TIMEOUTSECONDS}} {{end}}{{if .SECURITY_ENABLELOGIN}}SECURITY_ENABLELOGIN={{.SECURITY_ENABLELOGIN}} {{end}}{{if .POLICIES_ENABLED}}POLICIES_ENABLED={{.POLICIES_ENABLED}} {{end}}cmd /c ".\gradlew.bat :stirling-pdf:bootRun"'
         platforms: [windows]
-      - cmd: '{{if .AIENGINE_URL}}AIENGINE_URL={{.AIENGINE_URL}} AIENGINE_ENABLED={{.AIENGINE_ENABLED}} AIENGINE_TIMEOUTSECONDS={{.AIENGINE_TIMEOUTSECONDS}} {{end}}{{if .SECURITY_ENABLELOGIN}}SECURITY_ENABLELOGIN={{.SECURITY_ENABLELOGIN}} {{end}}./gradlew :stirling-pdf:bootRun'
+      - cmd: '{{if .AIENGINE_URL}}AIENGINE_URL={{.AIENGINE_URL}} AIENGINE_ENABLED={{.AIENGINE_ENABLED}} AIENGINE_TIMEOUTSECONDS={{.AIENGINE_TIMEOUTSECONDS}} {{end}}{{if .SECURITY_ENABLELOGIN}}SECURITY_ENABLELOGIN={{.SECURITY_ENABLELOGIN}} {{end}}{{if .POLICIES_ENABLED}}POLICIES_ENABLED={{.POLICIES_ENABLED}} {{end}}./gradlew :stirling-pdf:bootRun'
         platforms: [linux, darwin]
 
   dev:bundled:

.taskfiles/frontend.yml

@@ -404,10 +404,22 @@ tasks:
 
   test:
     desc: "Run tests"
+    cmds:
+      - task: test:editor
+      - task: test:portal
+
+  test:editor:
+    desc: "Run editor tests"
     deps: [prepare]
     cmds:
       - npx vitest run --root editor
 
+  test:portal:
+    desc: "Run portal tests"
+    deps: [prepare]
+    cmds:
+      - npx vitest run --root portal
+
   test:watch:
     desc: "Run tests in watch mode"
     deps: [prepare]

Taskfile.yml

@@ -90,6 +90,7 @@ tasks:
         vars:
           PORT: '{{.BACKEND_PORT}}'
           SECURITY_ENABLELOGIN: "true"
+          POLICIES_ENABLED: "true"
       - task: frontend:dev:portal
         vars:
           PORT: '{{.PORTAL_PORT}}'
@@ -109,6 +110,7 @@ tasks:
         vars:
           PORT: '{{.BACKEND_PORT}}'
           SECURITY_ENABLELOGIN: "true"
+          POLICIES_ENABLED: "true"
       - task: frontend:dev:portal
         vars:
           PORT: '{{.PORTAL_PORT}}'
@@ -135,6 +137,7 @@ tasks:
         vars:
           PORT: '{{.BACKEND_PORT}}'
           SECURITY_ENABLELOGIN: "true"
+          POLICIES_ENABLED: "true"
       - task: frontend:dev:proprietary
         vars:
           PORT: '{{.EDITOR_PORT}}'
@@ -210,6 +213,7 @@ tasks:
         vars:
           PORT: '{{.BACKEND_PORT}}'
           SECURITY_ENABLELOGIN: "true"
+          POLICIES_ENABLED: "true"
       - task: frontend:preview:portal:proxy
         vars:
           PORT: '{{.PROXY_PORT}}'

app/common/src/main/java/stirling/software/common/model/ApplicationProperties.java

@@ -206,6 +206,11 @@ public static class FileReadiness {
 
     @Data
     public static class Policies {
+        /**
+         * Master switch for the policy + sources subsystem (the PAYG-metered automation surface).
+         */
+        private boolean enabled = false;
+
         /**
          * Absolute directories that policy folder input sources and output sinks may read from or
          * write to. Empty (the default) disables folder access entirely, so a policy can never be

app/proprietary/src/main/java/stirling/software/proprietary/policy/config/FolderAccessGuard.java

@@ -4,14 +4,16 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Optional;
 
-import org.springframework.context.annotation.Profile;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
 import org.springframework.core.env.Environment;
 import org.springframework.stereotype.Component;
 
 import stirling.software.common.configuration.InstallationPathConfig;
 import stirling.software.common.model.ApplicationProperties;
 import stirling.software.proprietary.policy.model.Policy;
+import stirling.software.proprietary.policy.source.SourceStore;
 
 /**
  * Authority on which filesystem locations a policy may read/write. Checked at save time and again
@@ -28,20 +30,25 @@
  * defended: an operator who roots an allowlist on a symlink to a sensitive location is trusted.
  */
 @Component
-@Profile("saas")
+@ConditionalOnBooleanProperty(name = "policies.enabled")
 public class FolderAccessGuard {
 
     public static final String FOLDER_TYPE = "folder";
 
     private final boolean saasActive;
     private final List<Path> allowedRoots;
     private final List<Path> protectedRoots;
+    private final SourceStore sourceStore;
 
-    public FolderAccessGuard(ApplicationProperties applicationProperties, Environment environment) {
+    public FolderAccessGuard(
+            ApplicationProperties applicationProperties,
+            Environment environment,
+            SourceStore sourceStore) {
         this.saasActive = Arrays.asList(environment.getActiveProfiles()).contains("saas");
         this.allowedRoots =
                 normalizeAll(applicationProperties.getPolicies().getAllowedFolderRoots());
         this.protectedRoots = List.of(normalize(Path.of(InstallationPathConfig.getConfigPath())));
+        this.sourceStore = sourceStore;
     }
 
     /** Returns the normalised absolute path; throws if not permitted. */
@@ -72,7 +79,10 @@ public Path requirePermitted(Path dir) {
     /** Whether this policy touches a folder source/sink, and so is subject to these rules. */
     public boolean usesFolderAccess(Policy policy) {
         boolean readsFolder =
-                policy.sources().stream().anyMatch(spec -> FOLDER_TYPE.equals(spec.type()));
+                policy.sourceIds().stream()
+                        .map(sourceStore::get)
+                        .flatMap(Optional::stream)
+                        .anyMatch(source -> FOLDER_TYPE.equals(source.type()));
         boolean writesFolder =
                 policy.output() != null && FOLDER_TYPE.equals(policy.output().type());
         return readsFolder || writesFolder;

app/proprietary/src/main/java/stirling/software/proprietary/policy/config/PolicyAccessGuard.java

@@ -3,14 +3,15 @@
 import java.util.List;
 import java.util.Objects;
 
-import org.springframework.context.annotation.Profile;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
 import org.springframework.stereotype.Component;
 
 import lombok.RequiredArgsConstructor;
 
 import stirling.software.common.model.ApplicationProperties;
 import stirling.software.common.service.UserServiceInterface;
 import stirling.software.proprietary.policy.model.Policy;
+import stirling.software.proprietary.policy.store.PolicyStore;
 
 /**
  * Policies are scoped to a team: a user may view, run, edit, and delete only the policies belonging
@@ -22,7 +23,7 @@
  */
 @Component
 @RequiredArgsConstructor
-@Profile("saas")
+@ConditionalOnBooleanProperty(name = "policies.enabled")
 public class PolicyAccessGuard {
 
     private final UserServiceInterface userService;
@@ -47,13 +48,16 @@ public boolean canAccess(Policy policy) {
         return Objects.equals(policy.teamId(), policyManagementAuthority.currentUserTeamId());
     }
 
-    /** The subset of {@code policies} scoped to the current user's team. */
-    public List<Policy> visible(List<Policy> policies) {
+    /**
+     * The policies visible to the caller: their whole team's, loaded scoped rather than fetched
+     * globally and filtered, so on SaaS it never pulls another team's policies into memory. Login
+     * disabled (single-user) returns everything.
+     */
+    public List<Policy> visibleFrom(PolicyStore store) {
         if (!enforced()) {
-            return policies;
+            return store.all();
         }
-        Long teamId = policyManagementAuthority.currentUserTeamId();
-        return policies.stream().filter(policy -> Objects.equals(policy.teamId(), teamId)).toList();
+        return store.findByTeam(policyManagementAuthority.currentUserTeamId());
     }
 
     private boolean enforced() {

app/proprietary/src/main/java/stirling/software/proprietary/policy/controller/PolicyController.java

@@ -6,7 +6,7 @@
 import java.util.List;
 import java.util.Map;
 
-import org.springframework.context.annotation.Profile;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
 import org.springframework.core.io.FileSystemResource;
 import org.springframework.core.io.Resource;
 import org.springframework.http.HttpStatus;
@@ -53,7 +53,10 @@
 import stirling.software.proprietary.policy.model.PolicyRunStatus;
 import stirling.software.proprietary.policy.model.PolicyRunView;
 import stirling.software.proprietary.policy.progress.PolicyProgressListener;
+import stirling.software.proprietary.policy.source.SourceAccessGuard;
+import stirling.software.proprietary.policy.source.SourceStore;
 import stirling.software.proprietary.policy.store.PolicyStore;
+import stirling.software.proprietary.policy.trigger.PolicyTriggerManager;
 
 /**
  * Policy CRUD plus pipeline runs (stored or ad-hoc). Runs are async: returns a run id, poll {@code
@@ -65,15 +68,18 @@
 @Hidden
 @RequiredArgsConstructor
 @Tag(name = "Policies", description = "Run tool pipelines on the backend")
-@Profile("saas")
+@ConditionalOnBooleanProperty(name = "policies.enabled")
 public class PolicyController {
 
     private final PolicyRunner policyRunner;
     private final PolicyRunRegistry runRegistry;
     private final PolicyStore policyStore;
+    private final SourceStore sourceStore;
+    private final SourceAccessGuard sourceAccessGuard;
     private final PolicyValidator policyValidator;
     private final PolicyAccessGuard policyAccessGuard;
     private final PolicyManagementAuthority policyManagementAuthority;
+    private final PolicyTriggerManager policyTriggerManager;
     private final ApplicationProperties applicationProperties;
     private final TempFileManager tempFileManager;
     private final JobOwnershipService jobOwnershipService;
@@ -187,12 +193,33 @@ private boolean ownedByCurrentUser(String runId) {
     public ResponseEntity<Policy> savePolicy(@RequestBody Policy policy) {
         requirePolicyEditingAllowed();
         Policy owned = resolveOwnership(policy);
+        requireAccessibleSources(owned);
         try {
             policyValidator.validate(owned);
         } catch (IllegalArgumentException e) {
             throw new ResponseStatusException(HttpStatus.BAD_REQUEST, e.getMessage());
         }
-        return ResponseEntity.ok(policyStore.save(owned));
+        Policy saved = policyStore.save(owned);
+        // Re-sync trigger registrations now so a new/changed folder-watch policy starts being
+        // watched immediately instead of after the next reconcile sweep.
+        policyTriggerManager.notifyPoliciesChanged();
+        return ResponseEntity.ok(saved);
+    }
+
+    /**
+     * Every {@code sourceId} a policy references must resolve to a source in the caller's team, so
+     * a client can neither reference a non-existent source nor reach across teams to use another
+     * team's connection. A bad reference is a client error.
+     */
+    private void requireAccessibleSources(Policy policy) {
+        for (String sourceId : policy.sourceIds()) {
+            boolean accessible =
+                    sourceStore.get(sourceId).filter(sourceAccessGuard::canAccess).isPresent();
+            if (!accessible) {
+                throw new ResponseStatusException(
+                        HttpStatus.BAD_REQUEST, "Unknown or inaccessible source: " + sourceId);
+            }
+        }
     }
 
     /**
@@ -225,7 +252,7 @@ private static Policy withOwnerAndTeam(Policy policy, String owner, Long teamId)
                 owner,
                 policy.enabled(),
                 policy.trigger(),
-                policy.sources(),
+                policy.sourceIds(),
                 policy.steps(),
                 policy.output(),
                 teamId);
@@ -257,7 +284,7 @@ private void requirePolicyEditingAllowed() {
             summary = "List policies",
             description = "Lists the policies belonging to the caller's team.")
     public List<Policy> listPolicies() {
-        return policyAccessGuard.visible(policyStore.all());
+        return policyAccessGuard.visibleFrom(policyStore);
     }
 
     @GetMapping("/{policyId}")
@@ -278,6 +305,9 @@ public ResponseEntity<Void> deletePolicy(@PathVariable String policyId) {
         boolean accessible =
                 policyStore.get(policyId).filter(policyAccessGuard::canAccess).isPresent();
         if (accessible && policyStore.delete(policyId)) {
+            // Cancel any now-orphaned folder watch promptly rather than leaving the WatchKey open
+            // until the next reconcile sweep.
+            policyTriggerManager.notifyPoliciesChanged();
             return ResponseEntity.noContent().build();
         }
         return ResponseEntity.notFound().build();

app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyEngine.java

@@ -9,7 +9,7 @@
 import java.util.concurrent.ExecutorService;
 
 import org.slf4j.MDC;
-import org.springframework.context.annotation.Profile;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
 import org.springframework.core.io.Resource;
 import org.springframework.http.ResponseEntity;
 import org.springframework.security.core.Authentication;
@@ -54,7 +54,7 @@
 @Slf4j
 @Service
 @RequiredArgsConstructor
-@Profile("saas")
+@ConditionalOnBooleanProperty(name = "policies.enabled")
 public class PolicyEngine {
 
     // Admission weight for one run. Weighted heavy: a run chains many tools and holds intermediate

app/proprietary/src/main/java/stirling/software/proprietary/policy/engine/PolicyRunRegistry.java

@@ -9,7 +9,7 @@
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 
-import org.springframework.context.annotation.Profile;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty;
 import org.springframework.stereotype.Service;
 
 import jakarta.annotation.PreDestroy;
@@ -29,7 +29,7 @@
  */
 @Slf4j
 @Service
-@Profile("saas")
+@ConditionalOnBooleanProperty(name = "policies.enabled")
 public class PolicyRunRegistry {
 
     private final Map<String, PolicyRun> runs = new ConcurrentHashMap<>();