Merge pull request #31 from Asymptote-Labs/shukan/elastic-integration

Add Elastic endpoint integration pack

Author: shukieshah

CLAUDE.md

@@ -19,7 +19,7 @@ Do not recreate or depend on removed `asymptote` mirror trees. Keep new work foc
 - Preserve the local-only product posture. The public Beacon build should not require a hosted account, remote policy fetch, hosted dashboard, or external network dependency during normal hook execution.
 - Do not add dependency vulnerability scanning, OSV/GHSA lookups, package remediation, or other vulnerability-enforcement flows to the public hook path.
 - Do not add broad runtime enforcement unless explicitly requested. Current control behavior is limited to hook-native approvals/denials exposed by supported agent runtimes.
-- Keep direct destination support scoped to local JSONL/Wazuh unless explicitly requested. Other SIEM/observability systems should consume the local output through customer-managed forwarding.
+- Keep direct destination support scoped to local JSONL/Wazuh unless explicitly requested. Elastic support is a file-tailing pack over local JSONL; Beacon itself must not store Elastic credentials or require a hosted Elastic dependency.
 - Default content retention is `full`: configured prompt text, command output, raw tool inputs, raw OTLP attributes, and raw diffs may be written to local or customer-controlled logs, still subject to local redaction and size limits where supported. Keep `metadata` and `redacted` modes available for stricter deployments.
 
 ## Telemetry Scope
@@ -30,12 +30,13 @@ Supported runtime surfaces today:
 - Cursor hook telemetry for sessions, prompt submission, tool use, command execution, MCP-like tool activity, approval decisions, and file edits where hook payloads expose those fields.
 - Claude Cowork admin-configured OpenTelemetry setup guidance and local validation.
 - `beaconjson` OpenTelemetry Collector exporter that converts OTLP logs, traces, metrics, and resource attributes into Beacon endpoint JSONL.
+- Elasticsearch/Filebeat content pack generation for forwarding local Beacon JSONL into customer-managed Elastic deployments or the bundled loopback-only development stack.
 - A local-only dashboard served by `beacon endpoint dashboard`, bound to loopback by default and backed by the runtime JSONL log.
 
 Current non-goals unless explicitly requested:
 
 - Kernel/process monitoring, EDR replacement, shell history scraping, cloud audit ingestion, browser/SaaS telemetry, credential-use attribution, and MCP configuration inventory.
-- Direct hosted integrations for Datadog, Splunk, Elastic, Snowflake, Chronicle, Panther, or other SIEM destinations.
+- Direct hosted integrations for Datadog, Snowflake, Chronicle, Panther, or other SIEM destinations beyond explicitly supported local/customer-managed forwarding patterns.
 - Dependency vulnerability scanning or package security remediation.
 
 ## Common Commands

README.md

@@ -40,7 +40,7 @@ locally.
 
 Beacon is built to be easy to deploy for Security and IT teams through
 [MDM deployment](https://docs.asymptotelabs.ai/cli/security-it-teams) and to
-connect to Wazuh, Splunk HEC, or customer-managed SIEM pipelines, while
+connect to Wazuh, Elastic, Splunk HEC, or customer-managed SIEM pipelines, while
 remaining visibility-first and local-first during normal endpoint collection.
 
 ## High-Level Architecture
@@ -57,7 +57,7 @@ leaving forwarding under customer control.
 - **Beacon endpoint layer:** Local processing normalizes events, applies
   retention and redaction settings, and writes durable endpoint telemetry.
 - **Output layer:** Teams inspect events in the local dashboard, retain JSONL,
-  or forward records into Wazuh, Splunk HEC, and customer-managed SIEM
+  or forward records into Wazuh, Elastic, Splunk HEC, and customer-managed SIEM
   pipelines.
 
 Beacon filters generic process and runtime metrics, such as Node.js event loop,
@@ -82,6 +82,23 @@ records during rollout, testing, and investigations.
   <img src="images/dashboard-log-search.png" alt="Beacon dashboard log search" width="860">
 </p>
 
+## Elastic
+
+Beacon ships an Elastic content pack for teams that want to search endpoint
+events in Elasticsearch and Kibana without giving Beacon cluster credentials.
+The pack tails the same local `runtime.jsonl` file with Filebeat or standalone
+Elastic Agent, installs ECS-oriented templates and an ingest pipeline, and
+includes starter Kibana assets.
+
+```bash
+beacon endpoint elastic install-pack --output ./beacon-elastic-pack
+beacon endpoint elastic up --pack-dir ./beacon-elastic-pack
+```
+
+The local stack binds Elasticsearch and Kibana to loopback. Existing self-managed
+or Elastic Cloud deployments can use the same pack by pointing Filebeat at their
+cluster with `ES_HOSTS` and `ES_API_KEY`.
+
 ## Start Here
 
 - [Beacon CLI docs](https://docs.asymptotelabs.ai/cli) — full documentation index.

cli/beacon/cmd/endpoint.go

@@ -1,15 +1,20 @@
 package cmd
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
 	"strings"
 
 	"github.com/spf13/cobra"
 
 	endpointconfig "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/config"
 	"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/dashboard"
+	"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/elastic"
 	"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/harness"
 	endpointhooks "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/hooks"
 	"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/integrations/cowork"
@@ -41,6 +46,7 @@ var endpointOpts struct {
 	coworkNgrok              bool
 	coworkOpen               bool
 	coworkSince              string
+	elasticPackDir           string
 	hookLevel                string
 	contentRetention         string
 	splunkHECEndpoint        string
@@ -108,6 +114,11 @@ var endpointWazuhCmd = &cobra.Command{
 	Short: "Manage Wazuh integration content",
 }
 
+var endpointElasticCmd = &cobra.Command{
+	Use:   "elastic",
+	Short: "Manage Elasticsearch integration content",
+}
+
 var endpointIntegrationsCmd = &cobra.Command{
 	Use:   "integrations",
 	Short: "Manage admin-configured endpoint integrations",
@@ -230,6 +241,40 @@ var endpointWazuhValidateCmd = &cobra.Command{
 	RunE:         runEndpointWazuhValidate,
 }
 
+var endpointElasticPrintConfigCmd = &cobra.Command{
+	Use:   "print-config",
+	Short: "Print a Filebeat input for Beacon endpoint events",
+	Run: func(cmd *cobra.Command, args []string) {
+		cfg := loadOrDefaultConfig()
+		fmt.Print(elastic.InputSnippet(cfg.LogPath))
+	},
+}
+
+var endpointElasticInstallPackCmd = &cobra.Command{
+	Use:          "install-pack",
+	Short:        "Write Elasticsearch templates, pipeline, and Filebeat content to a directory",
+	SilenceUsage: true,
+	RunE:         runEndpointElasticInstallPack,
+}
+
+var endpointElasticUpCmd = &cobra.Command{
+	Use:          "up",
+	Short:        "Start a local Elasticsearch, Kibana, and Filebeat stack",
+	SilenceUsage: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runEndpointElasticUp(cmd.Context())
+	},
+}
+
+var endpointElasticDownCmd = &cobra.Command{
+	Use:          "down",
+	Short:        "Stop the local Elasticsearch stack",
+	SilenceUsage: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runEndpointElasticDown(cmd.Context())
+	},
+}
+
 func init() {
 	rootCmd.AddCommand(endpointCmd)
 
@@ -240,11 +285,16 @@ func init() {
 	endpointCmd.AddCommand(endpointRepairCmd)
 	endpointCmd.AddCommand(endpointDashboardCmd)
 	endpointCmd.AddCommand(endpointWazuhCmd)
+	endpointCmd.AddCommand(endpointElasticCmd)
 	endpointCmd.AddCommand(endpointIntegrationsCmd)
 	endpointCmd.AddCommand(endpointHooksCmd)
 	endpointWazuhCmd.AddCommand(endpointWazuhPrintConfigCmd)
 	endpointWazuhCmd.AddCommand(endpointWazuhInstallPackCmd)
 	endpointWazuhCmd.AddCommand(endpointWazuhValidateCmd)
+	endpointElasticCmd.AddCommand(endpointElasticPrintConfigCmd)
+	endpointElasticCmd.AddCommand(endpointElasticInstallPackCmd)
+	endpointElasticCmd.AddCommand(endpointElasticUpCmd)
+	endpointElasticCmd.AddCommand(endpointElasticDownCmd)
 	endpointIntegrationsCmd.AddCommand(endpointCoworkCmd)
 	endpointHooksCmd.AddCommand(endpointHooksInstallCmd)
 	endpointHooksCmd.AddCommand(endpointHooksUninstallCmd)
@@ -294,6 +344,14 @@ func init() {
 	endpointWazuhValidateCmd.Flags().BoolVar(&endpointOpts.userMode, "user", true, "Use per-user endpoint paths")
 	endpointWazuhValidateCmd.Flags().BoolVar(&endpointOpts.systemMode, "system", false, "Use system endpoint paths and launch daemon")
 	endpointWazuhValidateCmd.Flags().StringVar(&endpointOpts.logPath, "log-path", "", "Runtime JSONL log path")
+	for _, c := range []*cobra.Command{endpointElasticPrintConfigCmd, endpointElasticInstallPackCmd, endpointElasticUpCmd, endpointElasticDownCmd} {
+		c.Flags().BoolVar(&endpointOpts.userMode, "user", true, "Use per-user endpoint paths")
+		c.Flags().BoolVar(&endpointOpts.systemMode, "system", false, "Use system endpoint paths and launch daemon")
+		c.Flags().StringVar(&endpointOpts.logPath, "log-path", "", "Runtime JSONL log path")
+	}
+	endpointElasticInstallPackCmd.Flags().StringVar(&endpointOpts.outputDir, "output", "", "Output directory for Elasticsearch content pack")
+	endpointElasticUpCmd.Flags().StringVar(&endpointOpts.elasticPackDir, "pack-dir", elastic.DefaultOutputDir, "Elasticsearch pack directory")
+	endpointElasticDownCmd.Flags().StringVar(&endpointOpts.elasticPackDir, "pack-dir", elastic.DefaultOutputDir, "Elasticsearch pack directory")
 	for _, c := range []*cobra.Command{endpointCoworkPrintConfigCmd, endpointCoworkSetupCmd, endpointCoworkStatusCmd, endpointCoworkValidateCmd} {
 		c.Flags().BoolVar(&endpointOpts.userMode, "user", true, "Use per-user endpoint paths")
 		c.Flags().BoolVar(&endpointOpts.systemMode, "system", false, "Use system endpoint paths and launch daemon")
@@ -472,6 +530,123 @@ func runEndpointWazuhValidate(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
+func runEndpointElasticInstallPack(cmd *cobra.Command, args []string) error {
+	cfg := loadOrDefaultConfig()
+	outputDir := endpointOpts.outputDir
+	if outputDir == "" {
+		outputDir = elastic.DefaultOutputDir
+	}
+	if err := elastic.InstallPack(outputDir, cfg.LogPath); err != nil {
+		return err
+	}
+	fmt.Printf("Elasticsearch content pack written to %s\n", outputDir)
+	return nil
+}
+
+func runEndpointElasticUp(ctx context.Context) error {
+	if runtime.GOOS != "darwin" {
+		return fmt.Errorf("beacon endpoint elastic up is currently macOS-only")
+	}
+	cfg := loadOrDefaultConfig()
+	logPath, err := filepath.Abs(cfg.LogPath)
+	if err != nil {
+		return err
+	}
+	packDir := endpointOpts.elasticPackDir
+	if packDir == "" {
+		packDir = elastic.DefaultOutputDir
+	}
+	if err := ensureElasticPack(packDir, logPath); err != nil {
+		return err
+	}
+	if err := ensureLogFile(logPath); err != nil {
+		return err
+	}
+	env := os.Environ()
+	env = append(env, "BEACON_LOG_DIR="+filepath.Dir(logPath))
+	if err := runDockerCompose(ctx, packDir, env, "up", "-d"); err != nil {
+		return err
+	}
+	fmt.Printf("Elasticsearch ready at http://localhost:%s\n", envDefault("BEACON_ELASTIC_ES_PORT", "9200"))
+	fmt.Printf("Kibana ready at http://localhost:%s\n", envDefault("BEACON_ELASTIC_KIBANA_PORT", "5601"))
+	fmt.Printf("Filebeat tailing %s\n", logPath)
+	return nil
+}
+
+func runEndpointElasticDown(ctx context.Context) error {
+	if runtime.GOOS != "darwin" {
+		return fmt.Errorf("beacon endpoint elastic down is currently macOS-only")
+	}
+	packDir := endpointOpts.elasticPackDir
+	if packDir == "" {
+		packDir = elastic.DefaultOutputDir
+	}
+	if _, err := os.Stat(filepath.Join(packDir, "docker-compose.yml")); os.IsNotExist(err) {
+		fmt.Printf("No Elasticsearch stack found for %s\n", packDir)
+		return nil
+	} else if err != nil {
+		return err
+	}
+	logPath, err := filepath.Abs(loadOrDefaultConfig().LogPath)
+	if err != nil {
+		return err
+	}
+	env := append(os.Environ(), "BEACON_LOG_DIR="+filepath.Dir(logPath))
+	if err := runDockerCompose(ctx, packDir, env, "down", "--remove-orphans"); err != nil {
+		return err
+	}
+	fmt.Printf("Elasticsearch stack stopped for %s\n", packDir)
+	return nil
+}
+
+func ensureElasticPack(packDir, logPath string) error {
+	if _, err := os.Stat(filepath.Join(packDir, "docker-compose.yml")); err == nil {
+		return nil
+	} else if !os.IsNotExist(err) {
+		return err
+	}
+	if err := elastic.InstallPack(packDir, logPath); err != nil {
+		return err
+	}
+	fmt.Printf("Elasticsearch content pack written to %s\n", packDir)
+	return nil
+}
+
+func ensureLogFile(path string) error {
+	if path == "" {
+		return nil
+	}
+	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+		return err
+	}
+	file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND, 0644)
+	if err != nil {
+		return err
+	}
+	return file.Close()
+}
+
+func runDockerCompose(ctx context.Context, dir string, env []string, args ...string) error {
+	if _, err := os.Stat(filepath.Join(dir, "docker-compose.yml")); err != nil {
+		return fmt.Errorf("docker-compose.yml not found in %s: %w", dir, err)
+	}
+	fullArgs := append([]string{"compose"}, args...)
+	cmd := exec.CommandContext(ctx, "docker", fullArgs...)
+	cmd.Dir = dir
+	cmd.Env = env
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
+func envDefault(name, fallback string) string {
+	value := strings.TrimSpace(os.Getenv(name))
+	if value == "" {
+		return fallback
+	}
+	return value
+}
+
 func runEndpointDashboard(cmd *cobra.Command, args []string) error {
 	cfg := loadOrDefaultConfig()
 	userMode := endpointUserMode()

cli/beacon/cmd/endpoint_test.go

@@ -1,6 +1,9 @@
 package cmd
 
 import (
+	"os"
+	"path/filepath"
+	"runtime"
 	"testing"
 
 	"github.com/spf13/cobra"
@@ -50,6 +53,80 @@ func TestEndpointCoworkSetupCommandRegistered(t *testing.T) {
 	}
 }
 
+func TestEndpointElasticCommandsRegistered(t *testing.T) {
+	for _, path := range [][]string{
+		{"elastic", "print-config"},
+		{"elastic", "install-pack"},
+		{"elastic", "up"},
+		{"elastic", "down"},
+	} {
+		cmd, _, err := endpointCmd.Find(path)
+		if err != nil {
+			t.Fatalf("Find %v returned error: %v", path, err)
+		}
+		if cmd == nil || cmd.Use != path[len(path)-1] {
+			t.Fatalf("elastic command %v not registered: %#v", path, cmd)
+		}
+	}
+	if endpointElasticInstallPackCmd.Flags().Lookup("output") == nil {
+		t.Fatal("elastic install-pack command missing --output flag")
+	}
+	if endpointElasticUpCmd.Flags().Lookup("pack-dir") == nil {
+		t.Fatal("elastic up command missing --pack-dir flag")
+	}
+	if endpointElasticDownCmd.Flags().Lookup("pack-dir") == nil {
+		t.Fatal("elastic down command missing --pack-dir flag")
+	}
+	for _, name := range []string{"user", "system", "log-path"} {
+		if endpointElasticDownCmd.Flags().Lookup(name) == nil {
+			t.Fatalf("elastic down command missing --%s flag", name)
+		}
+	}
+}
+
+func TestEnsureElasticPackDoesNotOverwriteExistingPack(t *testing.T) {
+	dir := t.TempDir()
+	composePath := filepath.Join(dir, "docker-compose.yml")
+	if err := os.WriteFile(composePath, []byte("custom compose"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	if err := ensureElasticPack(dir, "/tmp/beacon/runtime.jsonl"); err != nil {
+		t.Fatalf("ensureElasticPack returned error: %v", err)
+	}
+	got, err := os.ReadFile(composePath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if string(got) != "custom compose" {
+		t.Fatalf("ensureElasticPack overwrote existing pack: %s", got)
+	}
+}
+
+func TestRunEndpointElasticDownIgnoresMissingPack(t *testing.T) {
+	if runtime.GOOS != "darwin" {
+		t.Skip("elastic down is currently macOS-only")
+	}
+	oldPackDir := endpointOpts.elasticPackDir
+	endpointOpts.elasticPackDir = filepath.Join(t.TempDir(), "missing-pack")
+	t.Cleanup(func() {
+		endpointOpts.elasticPackDir = oldPackDir
+	})
+	if err := runEndpointElasticDown(t.Context()); err != nil {
+		t.Fatalf("runEndpointElasticDown returned error for missing pack: %v", err)
+	}
+}
+
+func TestEnvDefault(t *testing.T) {
+	t.Setenv("BEACON_TEST_VALUE", "")
+	if got := envDefault("BEACON_TEST_VALUE", "fallback"); got != "fallback" {
+		t.Fatalf("envDefault empty = %q", got)
+	}
+	t.Setenv("BEACON_TEST_VALUE", "  12345  ")
+	if got := envDefault("BEACON_TEST_VALUE", "fallback"); got != "12345" {
+		t.Fatalf("envDefault value = %q", got)
+	}
+}
+
 func TestEndpointCoworkValidateSinceFlagRegistered(t *testing.T) {
 	cmd, _, err := endpointCmd.Find([]string{"integrations", "claude-cowork", "validate"})
 	if err != nil {