Experiment: Restructure project and flatten dependency graph

.env.example

@@ -2,25 +2,37 @@ PORT: 8080
 ENABLE_TLS: false
 CERT_FILE: "/certs/server.crt"
 CERT_KEY_FILE: "/certs/server.key"
-MAX_EXECUTIONS=1000
+MAX_RUNS=1000
 SOARCA_ALLOWED_ORIGINS: "*"
 GIN_MODE: "release"
-MONGODB_URI: "mongodb://localhost:27017"
-DATABASE_NAME: "soarca"
-DB_USERNAME: "root"
-DB_PASSWORD: "rootpassword"
+# Storage. The scheme selects the backend.
+#   sqlite://soarca.db   file in the working directory (default)
+#   sqlite://:memory:    in-memory, wiped on restart
+#   postgres://soarca:soarca@localhost:5432/soarca?sslmode=disable
+DATABASE_URL: "sqlite://soarca.db"
 PLAYBOOK_API_LOG_LEVEL: trace
-DATABASE: "false"
 MAX_REPORTERS: "5"
 
 LOG_GLOBAL_LEVEL: "info"
 LOG_MODE: "development"
 LOG_FILE_PATH: ""
 LOG_FORMAT: "json"
 
-ENABLE_FINS: false
-MQTT_BROKER: "localhost"
-MQTT_PORT: 1883
+# Fin protocol (HTTP/JSON pull-based external executors, see
+# docs/adr/FIN-WEBHOOK-PROTOCOL-PROPOSAL.md). FIN_REGISTRATION_TOKEN gates
+# POST /fin/register; leaving it unset/empty disables Fin registration
+# entirely (fails closed).
+FIN_REGISTRATION_TOKEN: "dev-registration-token"
+FIN_POLL_INTERVAL_SECONDS: 5
+FIN_LONG_POLL_TIMEOUT_SECONDS: 25
+FIN_JOB_LEASE_SECONDS: 60
+
+# Hard deadline used for a Manual or Fin step when its own cacao.Step.timeout
+# is omitted/zero (see pkg/utils/timeout.go). Not part of the CACAO spec's
+# own defaults - chosen to be realistic for human-in-the-loop approvals and
+# Fin jobs doing real external work, rather than the spec's own
+# __ACTION_TIMEOUT__ example value of 60000ms (1 minute).
+DEFAULT_STEP_TIMEOUT_SECONDS: 600
 
 HTTP_SKIP_CERT_VALIDATION: false
 ### Integrations

.gitignore

@@ -22,3 +22,8 @@ docs/.hugo_build.lock
 **.hugo_build.lock
 
 certs
+
+# Local SQLite databases
+soarca.db
+soarca.db-shm
+soarca.db-wal

README.md

@@ -18,7 +18,7 @@ SOARCA was developed for research and innovation purposes and allows SOC, CERT a
 
 
 ## Software
-SOARCA is a security orchestrator that can ingest, validate and execute CACAOv2 security playbooks. These playbooks and the triggers for their execution are consumed via a JSON API. SOARCA comes with native http(s), SSH and OpenC2 capabilities to interface with external tools and data resources. These native capabilities can be extended via a dedicated MQTT interface, allowing developers to compile additional integrations according their needs.
+SOARCA is a security orchestrator that can ingest, validate and execute CACAOv2 security playbooks. These playbooks and the triggers for their execution are consumed via a JSON API. SOARCA comes with native http(s), SSH and OpenC2 capabilities to interface with external tools and data resources. These native capabilities can be extended via a dedicated Fin interface, allowing developers to compile additional integrations according their needs.
 
 Development is ongoing. The current version solely supports machine and command line interfaces, but a graphical user interface will be added in the foreseeable future. Furthermore, its current capability to run CACAOv2 playbooks sequentially will evolve towards the ability to run multiple playbooks in parallel. Such further developments will be announced and published on the SOARCA repository on Github.
 

cmd/soarca/main.go

@@ -4,9 +4,9 @@ import (
 	"fmt"
 
 	api "soarca/api"
-	"soarca/internal/controller"
+	"soarca/internal/app"
 	"soarca/internal/logger"
-	"soarca/pkg/api/status"
+	"soarca/internal/transport/http/handlers/status"
 	"soarca/pkg/utils"
 
 	"github.com/joho/godotenv"
@@ -52,7 +52,7 @@ func main() {
 
 	// Version is only available here
 	status.SetVersion(Version)
-	err = controller.Initialize()
+	err = app.Run()
 	if err != nil {
 		log.Fatal("Something Went wrong with setting-up the app, msg: ", err)
 		panic(err)

deployments/docker/soarca/docker-compose.yml

@@ -12,31 +12,6 @@ services:
         source: mongodb_data_container
         target: /data/db
 
-  mosquitto:
-    image: docker.io/eclipse-mosquitto
-    container_name: mosquitto
-    volumes:
-      - type: volume
-        source: mosquitto_config
-        target: /mosquitto/config
-      - type: volume
-        source: mosquitto_data
-        target: /mosquitto/data
-      - type: volume
-        source: mosquitto_log
-        target: /mosquitto/log
-    networks:
-      - mqtt-net
-    ports:
-      - target: 1883
-        published: 1883
-        protocol: tcp
-        mode: host
-      - target: 9001
-        published: 9001
-        protocol: tcp
-        mode: host
-
   soarca-gui:
     image: docker.io/cossas/soarca-gui:latest
     container_name: soarca_gui
@@ -77,9 +52,6 @@ services:
       LOG_MODE: "production"
       LOG_FILE_PATH: ""
       LOG_FORMAT: "json"
-      ENABLE_FINS: true
-      MQTT_BROKER: "mosquitto"
-      MQTT_PORT: 1883
       HTTP_SKIP_CERT_VALIDATION: false
       # Integrations:
       # The Hive
@@ -88,13 +60,11 @@ services:
       THEHIVE_API_BASE_URL: http://localhost:9000/api/v1/
     networks:
       - db-net
-      - mqtt-net
       - soarca-net
     ports:
       - 127.0.0.1:8080:8080
     depends_on:
       - mongodb_container
-      - mosquitto
 
   loki:
     image: grafana/loki:3.0.0
@@ -133,18 +103,9 @@ services:
 
 networks:
   db-net:
-  mqtt-net:
   soarca-net:
 
 volumes:
   mongodb_data_container:
-  mosquitto_config:
-    driver: local # Define the driver and options under the volume name
-    driver_opts:
-      type: none
-      device: ./config
-      o: bind
-  mosquitto_data:
-  mosquitto_log:
   loki_data_container:
   grafana_data_container:

docker-compose.yaml

@@ -1,15 +1,16 @@
 version: "3.7"
 services:
-  mongodb_container:
-    image: mongo:latest
-    container_name: mongo_soarca_stack
+  postgres:
+    image: postgres:17
+    container_name: postgres_soarca_stack
     environment:
-      MONGO_INITDB_ROOT_USERNAME: "root"
-      MONGO_INITDB_ROOT_PASSWORD: "rootpassword"
+      POSTGRES_USER: "soarca"
+      POSTGRES_PASSWORD: "soarca"
+      POSTGRES_DB: "soarca"
     networks:
       - db-net
     volumes:
-      - mongodb_data_container:/data/db
+      - postgres_data_container:/var/lib/postgresql/data
 
   cert-generator:
     image: alpine
@@ -41,12 +42,8 @@ services:
       CERT_KEY_FILE: "/app/certs/server.key"
       SOARCA_ALLOWED_ORIGINS: "*"
       GIN_MODE: "release"
-      MONGODB_URI: "mongodb://mongodb_container:27017"
-      DATABASE_NAME: "soarca"
-      DB_USERNAME: "root"
-      DB_PASSWORD: "rootpassword"
+      DATABASE_URL: "postgres://soarca:soarca@postgres:5432/soarca?sslmode=disable"
       PLAYBOOK_API_LOG_LEVEL: trace
-      DATABASE: "false"
       HTTP_SKIP_CERT_VALIDATION: false
       AUTH_ENABLED: false #OPTIONAL for OIDC Based auth
       OIDC_PROVIDER: "https://authentikuri:9443/application/o/soarca/"
@@ -55,7 +52,7 @@ services:
     ports:
       - 127.0.0.1:8080:8080
     depends_on:
-      - mongodb_container
+      - postgres
       - cert-generator
 
   loki:
@@ -82,6 +79,6 @@ networks:
   db-net:
 
 volumes:
-  mongodb_data_container:
+  postgres_data_container:
   certs_data_containter:
   loki_data_container:

docs/content/en/docs/concepts/_index.md

@@ -59,7 +59,7 @@ One can generate playbooks using LLMs using the [playbook generation guide](/doc
 
 ### SOARCA Fin(s): Extending the core capabilities
 
-SOARCA can be extended with custom extensions or rather so-called FIN (inspired by the majestic orca). A fin can be integrated within the SOARCA core. Technical descriptions of the components can be found [here](/docs/soarca-extensions/fin-protocol). Fins communicate with the SOARCA core using a pre-defined MQTT protocol. 
+SOARCA can be extended with custom extensions or rather so-called FIN (inspired by the majestic orca). A fin can be integrated within the SOARCA core. Fins communicate with the SOARCA core over a simple pull-based HTTP/JSON protocol. Technical descriptions of the components can be found [here](/docs/soarca-extensions).
 
 
 ## Join the SOARCA Community