diff --git a/.env.example b/.env.example index 83a5bc2f..7cf09466 100644 --- a/.env.example +++ b/.env.example @@ -2,15 +2,15 @@ PORT: 8080 ENABLE_TLS: false CERT_FILE: "/certs/server.crt" CERT_KEY_FILE: "/certs/server.key" -MAX_EXECUTIONS=1000 +MAX_RUNS=1000 SOARCA_ALLOWED_ORIGINS: "*" GIN_MODE: "release" -MONGODB_URI: "mongodb://localhost:27017" -DATABASE_NAME: "soarca" -DB_USERNAME: "root" -DB_PASSWORD: "rootpassword" +# Storage. The scheme selects the backend. +# sqlite://soarca.db file in the working directory (default) +# sqlite://:memory: in-memory, wiped on restart +# postgres://soarca:soarca@localhost:5432/soarca?sslmode=disable +DATABASE_URL: "sqlite://soarca.db" PLAYBOOK_API_LOG_LEVEL: trace -DATABASE: "false" MAX_REPORTERS: "5" LOG_GLOBAL_LEVEL: "info" @@ -18,9 +18,21 @@ LOG_MODE: "development" LOG_FILE_PATH: "" LOG_FORMAT: "json" -ENABLE_FINS: false -MQTT_BROKER: "localhost" -MQTT_PORT: 1883 +# Fin protocol (HTTP/JSON pull-based external executors, see +# docs/adr/FIN-WEBHOOK-PROTOCOL-PROPOSAL.md). FIN_REGISTRATION_TOKEN gates +# POST /fin/register; leaving it unset/empty disables Fin registration +# entirely (fails closed). +FIN_REGISTRATION_TOKEN: "dev-registration-token" +FIN_POLL_INTERVAL_SECONDS: 5 +FIN_LONG_POLL_TIMEOUT_SECONDS: 25 +FIN_JOB_LEASE_SECONDS: 60 + +# Hard deadline used for a Manual or Fin step when its own cacao.Step.timeout +# is omitted/zero (see pkg/utils/timeout.go). Not part of the CACAO spec's +# own defaults - chosen to be realistic for human-in-the-loop approvals and +# Fin jobs doing real external work, rather than the spec's own +# __ACTION_TIMEOUT__ example value of 60000ms (1 minute). +DEFAULT_STEP_TIMEOUT_SECONDS: 600 HTTP_SKIP_CERT_VALIDATION: false ### Integrations diff --git a/.gitignore b/.gitignore index 387a3367..ef98d3f2 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,8 @@ docs/.hugo_build.lock **.hugo_build.lock certs + +# Local SQLite databases +soarca.db +soarca.db-shm +soarca.db-wal diff --git a/README.md b/README.md index 792e29f4..5bf0439f 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ SOARCA was developed for research and innovation purposes and allows SOC, CERT a ## Software -SOARCA is a security orchestrator that can ingest, validate and execute CACAOv2 security playbooks. These playbooks and the triggers for their execution are consumed via a JSON API. SOARCA comes with native http(s), SSH and OpenC2 capabilities to interface with external tools and data resources. These native capabilities can be extended via a dedicated MQTT interface, allowing developers to compile additional integrations according their needs. +SOARCA is a security orchestrator that can ingest, validate and execute CACAOv2 security playbooks. These playbooks and the triggers for their execution are consumed via a JSON API. SOARCA comes with native http(s), SSH and OpenC2 capabilities to interface with external tools and data resources. These native capabilities can be extended via a dedicated Fin interface, allowing developers to compile additional integrations according their needs. Development is ongoing. The current version solely supports machine and command line interfaces, but a graphical user interface will be added in the foreseeable future. Furthermore, its current capability to run CACAOv2 playbooks sequentially will evolve towards the ability to run multiple playbooks in parallel. Such further developments will be announced and published on the SOARCA repository on Github. diff --git a/cmd/soarca/main.go b/cmd/soarca/main.go index e49a8559..23bf9a1b 100644 --- a/cmd/soarca/main.go +++ b/cmd/soarca/main.go @@ -4,9 +4,9 @@ import ( "fmt" api "soarca/api" - "soarca/internal/controller" + "soarca/internal/app" "soarca/internal/logger" - "soarca/pkg/api/status" + "soarca/internal/transport/http/handlers/status" "soarca/pkg/utils" "github.com/joho/godotenv" @@ -52,7 +52,7 @@ func main() { // Version is only available here status.SetVersion(Version) - err = controller.Initialize() + err = app.Run() if err != nil { log.Fatal("Something Went wrong with setting-up the app, msg: ", err) panic(err) diff --git a/deployments/docker/mqtt/config/mosquitto.conf b/deployments/docker/mqtt/config/mosquitto.conf deleted file mode 100644 index 024452f6..00000000 --- a/deployments/docker/mqtt/config/mosquitto.conf +++ /dev/null @@ -1,9 +0,0 @@ -listener 1883 -persistence true -persistence_location /mosquitto/data/ -log_dest file /mosquitto/log/mosquitto.log - -## Authentication ## -# By default, Mosquitto >=2.0 allows only authenticated connections. Change to true to enable anonymous connections. -allow_anonymous true -# password_file /mosquitto/config/password.txt \ No newline at end of file diff --git a/deployments/docker/mqtt/data/.gitkeep b/deployments/docker/mqtt/data/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/deployments/docker/mqtt/docker-compose.yml b/deployments/docker/mqtt/docker-compose.yml deleted file mode 100644 index b8a884ee..00000000 --- a/deployments/docker/mqtt/docker-compose.yml +++ /dev/null @@ -1,34 +0,0 @@ -version: '3.7' -services: - mosquitto: - image: eclipse-mosquitto - container_name: mosquitto - volumes: - - type: volume - source: mosquitto_config - target: /mosquitto/config - - type: volume - source: mosquitto_data - target: /mosquitto/data - - type: volume - source: mosquitto_log - target: /mosquitto/log - ports: - - target: 1883 - published: 1883 - protocol: tcp - mode: host - - target: 9001 - published: 9001 - protocol: tcp - mode: host - -volumes: - mosquitto_config: - driver: local # Define the driver and options under the volume name - driver_opts: - type: none - device: ./config - o: bind - mosquitto_data: - mosquitto_log: diff --git a/deployments/docker/mqtt/log/.gitkeep b/deployments/docker/mqtt/log/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/deployments/docker/soarca/config/mosquitto.conf b/deployments/docker/soarca/config/mosquitto.conf deleted file mode 100644 index 024452f6..00000000 --- a/deployments/docker/soarca/config/mosquitto.conf +++ /dev/null @@ -1,9 +0,0 @@ -listener 1883 -persistence true -persistence_location /mosquitto/data/ -log_dest file /mosquitto/log/mosquitto.log - -## Authentication ## -# By default, Mosquitto >=2.0 allows only authenticated connections. Change to true to enable anonymous connections. -allow_anonymous true -# password_file /mosquitto/config/password.txt \ No newline at end of file diff --git a/deployments/docker/soarca/docker-compose.yml b/deployments/docker/soarca/docker-compose.yml index 180bc84b..635584ae 100644 --- a/deployments/docker/soarca/docker-compose.yml +++ b/deployments/docker/soarca/docker-compose.yml @@ -12,31 +12,6 @@ services: source: mongodb_data_container target: /data/db - mosquitto: - image: docker.io/eclipse-mosquitto - container_name: mosquitto - volumes: - - type: volume - source: mosquitto_config - target: /mosquitto/config - - type: volume - source: mosquitto_data - target: /mosquitto/data - - type: volume - source: mosquitto_log - target: /mosquitto/log - networks: - - mqtt-net - ports: - - target: 1883 - published: 1883 - protocol: tcp - mode: host - - target: 9001 - published: 9001 - protocol: tcp - mode: host - soarca-gui: image: docker.io/cossas/soarca-gui:latest container_name: soarca_gui @@ -77,9 +52,6 @@ services: LOG_MODE: "production" LOG_FILE_PATH: "" LOG_FORMAT: "json" - ENABLE_FINS: true - MQTT_BROKER: "mosquitto" - MQTT_PORT: 1883 HTTP_SKIP_CERT_VALIDATION: false # Integrations: # The Hive @@ -88,13 +60,11 @@ services: THEHIVE_API_BASE_URL: http://localhost:9000/api/v1/ networks: - db-net - - mqtt-net - soarca-net ports: - 127.0.0.1:8080:8080 depends_on: - mongodb_container - - mosquitto loki: image: grafana/loki:3.0.0 @@ -133,18 +103,9 @@ services: networks: db-net: - mqtt-net: soarca-net: volumes: mongodb_data_container: - mosquitto_config: - driver: local # Define the driver and options under the volume name - driver_opts: - type: none - device: ./config - o: bind - mosquitto_data: - mosquitto_log: loki_data_container: grafana_data_container: diff --git a/docker-compose.yaml b/docker-compose.yaml index dbb82f2a..6f37193d 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,15 +1,16 @@ version: "3.7" services: - mongodb_container: - image: mongo:latest - container_name: mongo_soarca_stack + postgres: + image: postgres:17 + container_name: postgres_soarca_stack environment: - MONGO_INITDB_ROOT_USERNAME: "root" - MONGO_INITDB_ROOT_PASSWORD: "rootpassword" + POSTGRES_USER: "soarca" + POSTGRES_PASSWORD: "soarca" + POSTGRES_DB: "soarca" networks: - db-net volumes: - - mongodb_data_container:/data/db + - postgres_data_container:/var/lib/postgresql/data cert-generator: image: alpine @@ -41,12 +42,8 @@ services: CERT_KEY_FILE: "/app/certs/server.key" SOARCA_ALLOWED_ORIGINS: "*" GIN_MODE: "release" - MONGODB_URI: "mongodb://mongodb_container:27017" - DATABASE_NAME: "soarca" - DB_USERNAME: "root" - DB_PASSWORD: "rootpassword" + DATABASE_URL: "postgres://soarca:soarca@postgres:5432/soarca?sslmode=disable" PLAYBOOK_API_LOG_LEVEL: trace - DATABASE: "false" HTTP_SKIP_CERT_VALIDATION: false AUTH_ENABLED: false #OPTIONAL for OIDC Based auth OIDC_PROVIDER: "https://authentikuri:9443/application/o/soarca/" @@ -55,7 +52,7 @@ services: ports: - 127.0.0.1:8080:8080 depends_on: - - mongodb_container + - postgres - cert-generator loki: @@ -82,6 +79,6 @@ networks: db-net: volumes: - mongodb_data_container: + postgres_data_container: certs_data_containter: loki_data_container: diff --git a/docs/content/en/docs/concepts/_index.md b/docs/content/en/docs/concepts/_index.md index 10ab71af..b05007c1 100644 --- a/docs/content/en/docs/concepts/_index.md +++ b/docs/content/en/docs/concepts/_index.md @@ -59,7 +59,7 @@ One can generate playbooks using LLMs using the [playbook generation guide](/doc ### SOARCA Fin(s): Extending the core capabilities -SOARCA can be extended with custom extensions or rather so-called FIN (inspired by the majestic orca). A fin can be integrated within the SOARCA core. Technical descriptions of the components can be found [here](/docs/soarca-extensions/fin-protocol). Fins communicate with the SOARCA core using a pre-defined MQTT protocol. +SOARCA can be extended with custom extensions or rather so-called FIN (inspired by the majestic orca). A fin can be integrated within the SOARCA core. Fins communicate with the SOARCA core over a simple pull-based HTTP/JSON protocol. Technical descriptions of the components can be found [here](/docs/soarca-extensions). ## Join the SOARCA Community diff --git a/docs/content/en/docs/core-components/api-manual.md b/docs/content/en/docs/core-components/api-manual.md index 1e8dabe9..af8457bd 100644 --- a/docs/content/en/docs/core-components/api-manual.md +++ b/docs/content/en/docs/core-components/api-manual.md @@ -17,12 +17,20 @@ We will use HTTP status codes https://en.wikipedia.org/wiki/List_of_HTTP_status_ @startuml protocol Manual { GET /manual - GET /manual/{execution-id}/{step-id} - POST /manual/continue + GET /manual/{execution-id}/{step-execution-id} + PUT /manual/{execution-id}/{step-execution-id} } @enduml ``` +A pending manual command is identified by its `execution-id` and +`step-execution-id`, not `step-id` - a step invoked more than once during an +execution (e.g. a step inside a `while-condition` loop body, or, once +implemented, several parallel branches converging on the same step) mints a +fresh `step-execution-id` per invocation, so several pending commands can +legitimately share the same `step-id` at once. Use `GET /manual` to discover +which specific `step-execution-id` you need to act on when more than one is +pending for the same `step-id`. ### /manual The manual interaction endpoint for SOARCA @@ -44,10 +52,9 @@ None |execution_id |UUID |string |The id of the execution |playbook_id |UUID |string |The id of the CACAO playbook executed by the execution |step_id |UUID |string |The id of the step executed by the execution -|description |description of the step|string |The description from the workflow step -|command |command |string |The command for the agent either command -|command_is_base64 |true \| false |bool |Indicates if the command is in Base64 -|target |cacao agent-target |object |Map of [cacao agent-target](https://docs.oasis-open.org/cacao/security-playbooks/v2.0/cs01/security-playbooks-v2.0-cs01.html#_Toc152256509) with the target(s) of this command +|step_execution_id |UUID |string |The id of this specific step invocation. Distinguishes concurrent/repeated pending commands that share the same step_id (e.g. overlapping loop iterations) +|commands |list of commands |array |All commands of the step, in order. Each entry has `description`, `command`, and `command_is_base64` +|targets |list of manual targets |array |All targets of the step, in order. Each entry has `target` ([cacao agent-target](https://docs.oasis-open.org/cacao/security-playbooks/v2.0/cs01/security-playbooks-v2.0-cs01.html#_Toc152256509)) and `authentication` (resolved [cacao authentication information](https://docs.oasis-open.org/cacao/security-playbooks/v2.0/cs01/security-playbooks-v2.0-cs01.html#_Toc152256496), if any -- a human operator needs credentials to act manually) |out_args |cacao variables |dictionary |Map of [cacao variables](https://docs.oasis-open.org/cacao/security-playbooks/v2.0/cs01/security-playbooks-v2.0-cs01.html#_Toc152256555) handled in the step out args with current values and definitions @@ -59,17 +66,30 @@ None "execution_id" : "", "playbook_id" : "", "step_id" : "", - "command" : "", - "command_is_base64" : "false", - "targets" : { - "__target1__" : { - "type" : "", - "name" : "", + "step_execution_id" : "", + "commands" : [ + { "description" : "", - "location" : "<.>", - "agent_target_extensions" : {} + "command" : "", + "command_is_base64" : "false" } - }, + ], + "targets" : [ + { + "target" : { + "type" : "", + "name" : "", + "description" : "", + "location" : "<.>", + "agent_target_extensions" : {} + }, + "authentication" : { + "type" : "", + "username" : "", + "password" : "" + } + } + ], "out_args": { "" : { "type": "", @@ -91,8 +111,9 @@ General error --- -#### GET `/manual//` -Get pending manual actions objects that are currently waiting in SOARCA for specific execution. +#### GET `/manual//` +Get the pending manual command identified by this execution and step +execution invocation. ##### Call payload None @@ -108,10 +129,9 @@ None |execution_id |UUID |string |The id of the execution |playbook_id |UUID |string |The id of the CACAO playbook executed by the execution |step_id |UUID |string |The id of the step executed by the execution -|description |description of the step|string |The description from the workflow step -|command |command |string |The command for the agent either command -|command_is_base64 |true \| false |bool |Indicates if the command is in Base64 -|targets |cacao agent-target |dictionary |Map of [cacao agent-target](https://docs.oasis-open.org/cacao/security-playbooks/v2.0/cs01/security-playbooks-v2.0-cs01.html#_Toc152256509) with the target(s) of this command +|step_execution_id |UUID |string |The id of this specific step invocation +|commands |list of commands |array |All commands of the step, in order. Each entry has `description`, `command`, and `command_is_base64` +|targets |list of manual targets |array |All targets of the step, in order. Each entry has `target` ([cacao agent-target](https://docs.oasis-open.org/cacao/security-playbooks/v2.0/cs01/security-playbooks-v2.0-cs01.html#_Toc152256509)) and `authentication` (resolved [cacao authentication information](https://docs.oasis-open.org/cacao/security-playbooks/v2.0/cs01/security-playbooks-v2.0-cs01.html#_Toc152256496), if any -- a human operator needs credentials to act manually) |out_args |cacao variables |dictionary |Map of [cacao variables](https://docs.oasis-open.org/cacao/security-playbooks/v2.0/cs01/security-playbooks-v2.0-cs01.html#_Toc152256555) handled in the step out args with current values and definitions @@ -124,17 +144,30 @@ None "execution_id" : "", "playbook_id" : "", "step_id" : "", - "command" : "", - "command_is_base64" : "false", - "targets" : { - "__target1__" : { - "type" : "", - "name" : "", + "step_execution_id" : "", + "commands" : [ + { "description" : "", - "location" : "<.>", - "agent_target_extensions" : {} + "command" : "", + "command_is_base64" : "false" + } + ], + "targets" : [ + { + "target" : { + "type" : "", + "name" : "", + "description" : "", + "location" : "<.>", + "agent_target_extensions" : {} + }, + "authentication" : { + "type" : "", + "username" : "", + "password" : "" + } } - }, + ], "out_args": { "" : { "type": "", @@ -154,16 +187,20 @@ None 404/Not found with payload: General error -#### POST `/manual/continue` -Respond to manual command pending in SOARCA, if out_args are defined they must be filled in and returned in the payload body. Only value is required in the response of the variable. You can however return the entire object. If the object does not match the original out_arg, the call we be considered as failed. +#### PUT `/manual//` +Resolve the pending manual command identified by this execution and step +execution invocation. If out_args are defined they must be filled in and +returned in the payload body. Only value is required in the response of the +variable. You can however return the entire object. If the object does not +match the original out_arg, the call will be considered as failed. + +This is a PUT on the same resource `GET /manual/{execution-id}/{step-execution-id}` +identifies - the ids therefore live in the path, not the body. ##### Call payload |field |content |type | description | | ----------------- | --------------------- | ----------------- | ----------- | |type |execution-status |string |The type of this content -|execution_id |UUID |string |The id of the execution -|playbook_id |UUID |string |The id of the CACAO playbook executed by the execution -|step_id |UUID |string |The id of the step executed by the execution |response_status |enum |string |`success` indicates successfull fulfilment of the manual request. `failure` indicates failed satisfaction of the request |response_out_args |cacao variables |dictionary |Map of cacao variables names to cacao variable struct. Only name, type, and value are mandatory @@ -173,9 +210,6 @@ Respond to manual command pending in SOARCA, if out_args are defined they must b { "type" : "manual-step-response", - "execution_id" : "", - "playbook_id" : "", - "step_id" : "", "response_status" : "success | failure", "response_out_args": { "" : { @@ -198,4 +232,7 @@ Generic execution information ##### Error 400/BAD REQUEST with payload: -General error \ No newline at end of file +General error + +404/NOT FOUND with payload: +General error, if no pending command exists for this execution-id/step-execution-id diff --git a/docs/content/en/docs/core-components/executer.md b/docs/content/en/docs/core-components/executer.md index 23597e48..946f3a4d 100644 --- a/docs/content/en/docs/core-components/executer.md +++ b/docs/content/en/docs/core-components/executer.md @@ -100,7 +100,7 @@ The action executor consist of the following components - The capability selector - Native capabilities (command executors) -- MQTT capability to interact with: Fin capabilities (third-party executors) +- HTTP/JSON capability to interact with: Fin capabilities (third-party executors) The capability selector will select the implementation which is capable of executing the incoming command. There are native capabilities based on the CACAO `command-type-ov`: @@ -121,14 +121,14 @@ The capability selector will select the implementation which is capable of execu * yara #### Native capabilities -The executor will select a module that is capable of executing the command and pass the details to it. The capability selection is performed based on the agent type (see [Agent and Target Common Properties](https://docs.oasis-open.org/cacao/security-playbooks/v2.0/cs01/security-playbooks-v2.0-cs01.html#_Toc152256509) in the CACAO 2.0 spec). The convention is that the agent type must equal `soarca-`, e.g. `soarca-ssh` or `soarca-openc2-http`. +The executor will select a module that is capable of executing the command and pass the details to it. Capability selection is performed based on the agent's `type` (see [Agent and Target Common Properties](https://docs.oasis-open.org/cacao/security-playbooks/v2.0/cs01/security-playbooks-v2.0-cs01.html#_Toc152256509) in the CACAO 2.0 spec). The convention is that the agent type must equal `soarca-`, e.g. `soarca-ssh` or `soarca-openc2-http`. `name` on the agent definition is a free-text, human-readable label only; it plays no role in routing. The result of the step execution will be returned to the decomposer. A result can be either output variables or error status. -#### MQTT executor -> Fin capabilities -The Executor will put the command on the MQTT topic that is offered by the module. How a module handles this is described in the [module documentation](/docs/core-components/modules) and in the [fin documentation](/docs/soarca-extensions/). +#### Fin capabilities +SOARCA is extendable via Fins — external processes that implement a pull-based HTTP/JSON protocol to claim and execute jobs for a given capability `type`. See the [fin documentation](/docs/soarca-extensions/) for more information. This is being redesigned; the diagrams below describe SOARCA's native (in-process) capabilities. #### Component overview @@ -141,16 +141,9 @@ component Decomposer as parser package "Executor" { component SSH as exe2 component "HTTP-API" as exe1 - component MQTT as exe3 -} - -package "Fins" { - component "VirusTotal" as virustotal - component "E-mail Sender" as email } parser -- Executor -exe3 -- Fins : " MQTT topics" ``` #### Sequences diff --git a/docs/content/en/docs/core-components/modules.md b/docs/content/en/docs/core-components/modules.md index a6af5818..517467f1 100644 --- a/docs/content/en/docs/core-components/modules.md +++ b/docs/content/en/docs/core-components/modules.md @@ -353,8 +353,8 @@ class ManualCommand protocol ManualAPI { GET /manual - GET /manual/{exec-id}/{step-id} - POST /manual/continue + GET /manual/{exec-id}/{step-execution-id} + PUT /manual/{exec-id}/{step-execution-id} } interface ICapability{ @@ -553,6 +553,3 @@ This module does not define specific variables as input, but it requires one to } ``` --- - -## MQTT fin module -This module is used by SOARCA to communicate with fins (capabilities) see [fin documentation](/docs/soarca-extensions/) for more information diff --git a/docs/content/en/docs/core-components/soarca-application-design.md b/docs/content/en/docs/core-components/soarca-application-design.md index 4cf8c04c..1a985cc6 100644 --- a/docs/content/en/docs/core-components/soarca-application-design.md +++ b/docs/content/en/docs/core-components/soarca-application-design.md @@ -8,7 +8,7 @@ description: > --- ## Design decisions and core dependencies -To allow for fast execution and type-safe development SOARCA is developed in `go`. The application application can be deployed in `Docker`. Further dependencies are `MQTT` for the module system and `go-gin` for the REST API. +To allow for fast execution and type-safe development SOARCA is developed in `go`. The application application can be deployed in `Docker`. Further dependencies are `go-gin` for the REST API. The overview on this page is aimed to guide you through the SOARCA architecture and components as well as the main flow. diff --git a/docs/content/en/docs/getting-started/_index.md b/docs/content/en/docs/getting-started/_index.md index 36589463..b0619888 100644 --- a/docs/content/en/docs/getting-started/_index.md +++ b/docs/content/en/docs/getting-started/_index.md @@ -94,10 +94,6 @@ LOG_MODE: "development" LOG_FILE_PATH: "" LOG_FORMAT: "json" -ENABLE_FINS: false -MQTT_BROKER: "localhost" -MQTT_PORT: 1883 - HTTP_SKIP_CERT_VALIDATION: false {{< /tab >}} {{< /tabpane >}} diff --git a/docs/content/en/docs/installation-configuration/_index.md b/docs/content/en/docs/installation-configuration/_index.md index fc789b1b..2c6d1fc3 100644 --- a/docs/content/en/docs/installation-configuration/_index.md +++ b/docs/content/en/docs/installation-configuration/_index.md @@ -18,12 +18,10 @@ After completing the [Getting Started](/docs/getting-started/_index.md) setup fo | ENABLE_TLS | `false` | Enable TLS for secure communication. Default is `false`. | | CERT_FILE | `"/certs/server.crt"` | Path to the TLS certificate file. Default is `"/certs/server.crt"`. | | CERT_KEY_FILE | `"/certs/server.key"` | Path to the TLS certificate key file. Default is `"/certs/server.key"`. | -| MAX_EXECUTIONS | `1000` | The number of historical executions saved, including the current one. Default is `1000`. | +| MAX_RUNS | `1000` | The number of historical runs saved, including the current one. Default is `1000`. | | SOARCA_ALLOWED_ORIGINS | `*` | Set allowed origins for cross-origin requests. Default is `*`. | | GIN_MODE | `release` | Set the GIN mode. Default is `release`. | -| DATABASE | `false` | Set if you want to run with an external database. Default is `false`. | -| MONGODB_URI | `mongodb://localhost:27017` | Set the MongoDB URI. Default is `mongodb://localhost:27017`. | -| DATABASE_NAME | `soarca` | Set the MongoDB database name when using Docker. Default is `soarca`. | +| DATABASE_URL | `sqlite://soarca.db` | Database URL. Use `sqlite://:memory:` for in-memory use or a `postgres://` URL for PostgreSQL. | | DB_USERNAME | `root` | Set the MongoDB database user when using Docker. Default is `root`. | | DB_PASSWORD | `rootpassword` | Set the MongoDB database user password when using Docker. **Change this in production!** Default is `rootpassword`. | | PLAYBOOK_API_LOG_LEVEL | `trace` | Set the log level for the playbook API. Default is `trace`. | @@ -32,9 +30,6 @@ After completing the [Getting Started](/docs/getting-started/_index.md) setup fo | LOG_MODE | `development` | Set the logging mode. If `production`, `LOG_GLOBAL_LEVEL` is used for all modules. Default is `development`. | | LOG_FILE_PATH | `""` | Path to the logfile for all logging. Default is `""` (empty string). | | LOG_FORMAT | `json` | Set the logging format. Either `text` or `json`. Default is `json`. | -| ENABLE_FINS | `false` | Enable FINS in SOARCA. Default is `false`. | -| MQTT_BROKER | `localhost` | The broker address for SOARCA to connect to for communication with FINS. Default is `localhost`. | -| MQTT_PORT | `1883` | The port for the MQTT broker. Default is `1883`. | | HTTP_SKIP_CERT_VALIDATION | `false` | Set whether to skip certificate validation for HTTP connections. Default is `false`. | | VALIDATION_SCHEMA_URL | `""` | Set a custom validation schema to validate playbooks. Default is `""` to use the internal schema. **Note:** Changing this can heavily impact performance. | diff --git a/docs/content/en/docs/soarca-extensions/_index.md b/docs/content/en/docs/soarca-extensions/_index.md index 43bbf94a..cb2b318d 100644 --- a/docs/content/en/docs/soarca-extensions/_index.md +++ b/docs/content/en/docs/soarca-extensions/_index.md @@ -17,11 +17,9 @@ SOARCA features a set of [native capabilities](/docs/soarca-extensions/native-ca ## Extending the native capabilities -The native capabilities supported by SOARCA can be extended through a mechanism we named Fins. Your capability can be integrated with SOARCA by implementing the Fin protocol. This protocol regulates communication between SOARCA and the extension capabilities over an MQTT bus. - -MQTT is a lightweight messaging protocol with libraries written in various programming languages. To integrate with SOARCA, you can write your own implementation of the Fin protocol, or use our [python](https://www.python.org/) or [golang](https://go.dev/) libraries for easier integration. +The native capabilities supported by SOARCA can be extended through a mechanism we named Fins. Your capability can be integrated with SOARCA by implementing the Fin protocol. This protocol regulates communication between SOARCA and the extension capabilities over a simple, pull-based HTTP/JSON API — a Fin only ever makes outbound calls to SOARCA (register, then repeatedly poll for work and report results), so no inbound connectivity or message broker is required on the Fin side. ## Fin protocol -The underlying protocol for the SOARCA fins can be found [here](/docs/soarca-extensions/fin-protocol). +The underlying protocol for SOARCA Fins can be found [here](/docs/soarca-extensions/fin-protocol). diff --git a/docs/content/en/docs/soarca-extensions/fin-protocol.md b/docs/content/en/docs/soarca-extensions/fin-protocol.md index b1c62ce4..938218de 100644 --- a/docs/content/en/docs/soarca-extensions/fin-protocol.md +++ b/docs/content/en/docs/soarca-extensions/fin-protocol.md @@ -5,589 +5,319 @@ description: > categories: [extensions, architecture] tags: [fin] weight: 2 -date: 2023-01-05 +date: 2026-08-26 --- ## Goals -The goal of the protocol is to provide a simple and robust way to communicate between the SOARCA orchestrator and the capabilities (Fins) that can provide extra functions. -## MQTT -To allow for dynamic communication MQTT is used to provide the backbone for the fin communication. SOARCA can be configured using the environment to use MQTT or just run stand-alone. - -The Fin will use the protocol to register itself to SOARCA via the register message. Once register, it will communicate over the channel new channel designated by the fin UUID. - -Commands to a specific capability will be communicated of the capability UUID channel. - -## Messages -Messages defined in the protocol - -- ack -- nack -- register -- unregister -- command -- pause -- resume -- stop - -### legend - -|field |content |type |description -|field name have the `(optional)` key if the field is not required |content indication |type of the value could be string, int etc. |A description for the field to provide extra information and context - - - -### ack -The ack message is used to acknowledge messages. - - -|field | content | type | description | -| ---- | ------- | ---- | ----------- | -|type |ack |string |The ack message type -|message_id |UUID |string |message id that the ack is referring to - - -```plantuml -@startjson -{ - "type": "ack", - "message_id": "uuid" -} -@endjson -``` - -### nack -The nack message is used to non acknowledgements, message was unimplemented or unsuccessful. - - -|field | content | type | description | -| ---- | ------- | ---- | ----------- | -|type |nack |string |The ack message type -|message_id |UUID |string |message id that the nack is referring to - - -```plantuml -@startjson -{ - "type": "nack", - "message_id": "uuid" -} -@endjson -``` - - - - -### register -The message is used to register a fin to SOARCA. It has the following payload. - - -|field |content |type | description | -| ----------------- | --------------------- | ----------------- | ----------- | -|type |register |string |The register message type -|message_id |UUID |string |Message UUID -|fin_id |UUID |string |Fin uuid separate form the capability id -|Name |Name |string |Fin name -|protocol_version |version |string |Version information of the protocol in [semantic version](https://semver.org) schema e.g. 1.2.4-beta -|security |security information |[Security](#security) |Security information for protocol see security structure -|capabilities |list of capability structure |list of [capability structure](#capability-structure) |Capability structure information for protocol see security structure -|meta |meta dict |[Meta](#meta) |Meta information for the fin protocol structure - - - - -#### capability structure - -|field |content |type | description | -| ----------------- | ------------- | ------ | ----------- | -|capability_id |UUID |string |Capability id to identify the unique capability a fin can have multiple -|type |action | [workflow-step-type-enum](https://docs.oasis-open.org/cacao/security-playbooks/v2.0/cs01/security-playbooks-v2.0-cs01.html#_Toc152256479) | Most common is action -|name |name |string |capability name -|version |version |string |Version information of the Fin implementation used in [semantic version](https://semver.org) schema e.g. 1.2.4-beta -|step |step structure |[step structure](#step-structure) |Step to specify an example for the step so it can be queried in the SOARCA API -|agent |agent structure|[agent structure](#agent-structure) |Agent to specify the agent definition to match in playbooks for SOARCA - - -#### step structure -|field |content | type | description | -| ----------------- | ------------- | ----------------- | ----------- | -|type |action |string |Action type -|name |name |string |message id -|description |description |string |Description of the step -|external_references| |list of [external reference](https://docs.oasis-open.org/cacao/security-playbooks/v2.0/cs01/security-playbooks-v2.0-cs01.html#_Toc152256542) |References to external recourses to further enhance the step also see CACAO V2 10.9. -|command |command |string |Command to execute -|target |UUID |string |Target UUID cto execute command against - - -#### agent structure - -|field |content | type | description | -| ----------------- | ------------- | ----------------- | ----------- | -|type |soarca-fin |string |SOARCA Fin type, a custom type used to specify Fins -|name |name |string |SOARCA Fin name in the following form: `soarca-fin--`, this grantees the fin is unique - -```plantuml -@startjson +The goal of the protocol is to provide a simple and robust way to +communicate between the SOARCA orchestrator and the capabilities (Fins) +that can provide extra functions. Fins are external, independently-deployed +processes: they register once with SOARCA, then repeatedly poll for work, +execute it, and report the result back. All calls are outbound from the +Fin — no inbound connectivity, firewall holes, or message broker are +required on the Fin side. + +{{% alert title="Note" color="info" %}} +This replaces the previous MQTT-based Fin protocol. There is no migration +path: any existing MQTT-based Fin implementation is not compatible with +this protocol. +{{% /alert %}} + +## Transport and authentication + +The protocol is plain HTTP(S) + JSON. There is no separate framing or +message-envelope layer — each protocol "message" from the previous MQTT +design is now just the body of a regular HTTP request/response. + +Three separate credentials/schemes are involved, each scoped to a different +purpose: + +| Credential | Used for | Sent as | +| ---------- | -------- | ------- | +| `FIN_REGISTRATION_TOKEN` | one-time, gating `POST /fin/register` | `registration_token` field in the request body | +| `fin_token` | every other Fin-initiated call (poll/result/status/unregister) | `Authorization: Bearer ` header | +| SOARCA admin JWT | the read-only discovery endpoints (`GET /fin/`, `GET /fin/{fin_id}`) | `Authorization: Bearer ` header, same as the rest of the admin API | + +`FIN_REGISTRATION_TOKEN` is a coarse, instance-level shared secret +configured server-side and distributed to Fin operators out-of-band. If it +is not configured (empty), registration is disabled entirely — SOARCA fails +closed rather than silently accepting any registration attempt. + +`fin_token` is returned once, at registration (see below), and is expected +to be persisted locally by the Fin (e.g. in a local config file) so a +restarted Fin process can start polling again immediately, without +re-registering. SOARCA never stores the plaintext token — only a one-way +hash of it — so a database read or leak alone cannot recover a usable +credential. + +Fin registrations are itself persisted (database-backed, mirroring how +playbooks are persisted): SOARCA does not need Fins to re-register every +time it restarts or is updated. The in-memory job queue, by contrast, is +*not* persisted — SOARCA does not persist or resume in-flight executions +across a restart either, so persisting only the job queue would add +complexity for no real gain (see +[EXECUTION-MODEL.md](https://github.com/COSSAS/SOARCA/blob/main/docs/adr/EXECUTION-MODEL.md)). +A restart loses in-flight jobs the same way it loses everything else about +an in-flight execution — any Fin still holding a claimed job simply has its +next status ping/result submission rejected, and the step fails once its +own timeout elapses. + +## Endpoints + +| Method | Path | Auth | Purpose | +| ------ | ---- | ---- | ------- | +| `POST` | `/fin/register` | registration token | Register a new Fin identity and obtain a `fin_token` | +| `POST` | `/fin/poll` | fin_token | Long-poll for the next job matching this Fin's registered capability types | +| `PUT` | `/fin/jobs/{job_id}` | fin_token | Submit the result of a claimed job | +| `PATCH` | `/fin/jobs/{job_id}/status` | fin_token | Extend a claimed job's lease and check for a pending instruction (e.g. cancellation) | +| `DELETE` | `/fin/` | fin_token | Unregister the calling Fin itself - the fin_id is inferred from the token, never sent explicitly | +| `GET` | `/fin/` | admin JWT | List all currently-registered Fins and their capabilities | +| `GET` | `/fin/{fin_id}` | admin JWT | Look up a specific registered Fin by id | +| `DELETE` | `/fin/{fin_id}` | admin JWT | Forcibly remove any Fin's registration (e.g. one that is stale/offline and will never unregister itself) | + +The full request/response bodies are documented in the generated +[OpenAPI/Swagger reference](/docs/soarca-api/), under the `fin` tag. + +### Registering a Fin + +A Fin process declares one or more **capabilities** at registration time — +each capability has a `type` (the routing key playbook authors write into +`agent_definitions[...].type`), plus optional `description`, `version`, and +illustrative `step_examples` (full CACAO action steps, shown to playbook +authors to demonstrate how to invoke the capability — never interpreted or +validated by SOARCA itself). + +Multiple, independently-deployed Fin processes may register the same +capability `type`. SOARCA treats them as one interchangeable pool: any of +them may claim a job queued under that type, competing via long-poll +(load-balancing and failover across a pool is "whichever Fin happens to be +idle and polling", with no separate leader-election or assignment logic). + +```json +POST /fin/register { - "type": "register", - "message_id": "uuid", - "fin_id" : "uuid", - "name": "Fin name", - "protocol_version": "", - "security": { - "version": "0.0.0", - "channel_security": "plaintext" - }, + "registration_token": "", + "display_name": "example-ssh-fin", + "protocol_version": "1.0.0", "capabilities": [ { - "capability_id": "uuid", - "name": "ssh executer", - "version": "0.1.0", - "step": { - "type": "action", - "name": "", - "description": "", - "external_references": { - "name": "", - "...": "..." - }, - "command": "", - "target": "" - }, - "agent" : { - "soarca-fin--": { - "type": "soarca-fin", - "name": "soarca-fin---" + "type": "custom-ssh-fin", + "description": "SSH command execution", + "version": "0.1.0", + "step_examples": [ + { + "type": "action", + "name": "Restart the nginx service", + "agent": "custom-ssh-fin--f3f0194f-99e6-4966-8512-de3806fecfdf", + "commands": [ + { + "type": "manual", + "command": "sudo systemctl restart nginx" + } + ] } - } - + ] } - ], - "meta": { - - "timestamp": "string: ", - "sender_id": "uuid" - } + ] } -@endjson ``` - - - -### unregister -The message is used to unregister a fin to SOARCA. It has the following payload. - -|field |content |type | description | -| ----------------- | ------------- | ------ | ----------- | -|type |unregister |string |Unregister message type -|message_id |UUID |string |Message UUID -|capability_id |UUID |string |Capability id or null (either capability_id != null, fin_id != null or all == true need to be set) -|fin_id |UUID |string |Fin id or null (either capability_id != null, fin_id != null or all == true need to be set) -|all |bool |bool |True to address all fins to unregister otherwise false (either capability_id != null, fin_id != null or all == true need to be set) - -```plantuml -@startjson +```json +201 Created { - "type": "unregister", - "message_id": "uuid", - "capability_id" : "capability uuid", - "fin_id" : "fin uuid", - "all" : "true | false" + "fin_id": "", + "fin_token": "", + "poll_interval_seconds": 5, + "long_poll_timeout_seconds": 25, + "job_lease_seconds": 60 } -@endjson ``` -### command -The message is used to send a command from SOARCA. It has the following payload. - -|field |content |type | description | -| ----------------- | ------------- | ------ | ----------- | -|type |command |string |Command message type -|message_id |UUID |string |Message UUID -|command |command |[command substructure](#command-substructure) |command structure -|meta |meta dict |[Meta](#meta) |Meta information for the fin protocol structure +`poll_interval_seconds`/`long_poll_timeout_seconds`/`job_lease_seconds` are +server-chosen operational defaults, echoed back so a Fin implementation +doesn't need its own hardcoded copy of them. +### Polling for work -#### command substructure -|field |content |type | description | -| ----------------- | ------------- | ------ | ----------- | -|command |command |string |The command to be executed -|authentication `(optional)` |authentication information | [authentication information](https://docs.oasis-open.org/cacao/security-playbooks/v2.0/cs01/security-playbooks-v2.0-cs01.html#_Toc152256503) | CACAO authentication information -|context |cacao context |[Context](#context) | Context form the playbook -|variables |dict of variables |dict of [Variables](#variables) | From the playbook +A Fin repeatedly calls `POST /fin/poll`, authenticated with its +`fin_token`. SOARCA long-polls the request: it holds the connection open +until a job matching one of the Fin's registered capability types becomes +available, or `long_poll_timeout_seconds` elapses — whichever comes first. +An empty body plus `204 No Content` means "no work right now, just poll +again"; this is the expected, common case, not an error. - -```plantuml -@startjson +```json +POST /fin/poll { - "type": "command", - "message_id": "uuid", - "command": { - "command": "command", - "authentication": {"auth-uuid": "", - "timeout": "string: ", - "step_id": "uuid", - "playbook_id": "uuid", - "execution_id": "uuid" - }, - "variables": { - "____": { - "type": "", - "name": "____", - "description": "", - "value": "", - "constant": "", - "external": "" - }, - "____": { - "type": "", - "name": "____", - "description": "", - "value": "", - "constant": "", - "external": "" - } - } - }, - "meta": { - "timestamp": "string: ", - "sender_id": "uuid" - } + "concurrency_available": 1 } -@endjson ``` -### result -The message is used to send a response from the Fin to SOARCA. It has the following payload. - -|field |content |type | description | -| ----------------- | ------------- | ------ | ----------- | -|type |result |string |Unregister message type -|message_id |UUID |string |Message UUID -|result |result structure |[result structure](#result-structure)| The result of the execution -|meta |meta dict |[Meta](#meta) |Meta information for the fin protocol structure - - -#### result structure - -|field |content |type | description | -| ----------------- | ------------- | ------ | ----------- | -|state |succes or failure |string | The execution state of the playbook -|context |cacao context |[Context](#context) | Context form the playbook -|variables |dict of variables |dict of [variables](#variables) |Dictionary of CACAO compatible variables - - -```plantuml -@startjson -{ - "type": "result", - "message_id": "uuid", - "result": { - "state": "enum(success | failure)", - "context": { - "generated_on": "string: ", - "timeout": "string: ", - "step_id": "uuid", - "playbook_id": "uuid", - "execution_id": "uuid" +```json +200 OK +{ + "job": { + "job_id": "", + "execution_id": "", + "playbook_id": "playbook--...", + "step_id": "action--...", + "step_execution_id": "", + "capability_type": "custom-ssh-fin", + "lease_expires_in_seconds": 60, + "step": { + "name": "Restart the nginx service", + "description": "...", + "timeout": 60, + "delay": 0 }, - "variables": { - "____": { - "type": "", - "name": "____", - "description": "", - "value": "", - "constant": "", - "external": "" - }, - "____": { - "type": "", - "name": "____", - "description": "", - "value": "", - "constant": "", - "external": "" + "commands": [ + { "type": "manual", "command": "sudo systemctl restart nginx" } + ], + "targets": [ + { + "target": { "type": "ipv4-addr", "name": "web-01", "address": ["10.0.0.5"] }, + "authentication": { "type": "user-auth", "username": "deploy" } } - } - }, - "meta": { - "timestamp": "string: ", - "sender_id": "uuid" + ], + "variables": {} } } -@endjson -``` - - - -### control -|field |content |type | description | -| ----------------- | ------------- | ------ | ----------- | -|type |pause or resume or stop or progress |string |Message type -|message_id |UUID |string |message uuid -|capability_id |UUID |string |Capability uuid to control - -#### pause -The message is used to halt the further execution of the Fin. The following command will be responded to with a nack, unless it is resumed or stopped. - -```plantuml -@startjson -{ - "type": "pause", - "message_id" : "uuid", - "capability_id": "uuid" -} -@endjson ``` - -#### resume -The message is used to resume a paused Fin, the response will be an ack if ok or a nack when the Fin could not be resumed. - -```plantuml -@startjson -{ - "type": "resume", - "message_id" : "uuid", - "capability_id": "uuid" -} -@endjson -``` - -#### stop -The message is used to shut down the Fin. this will be responded to by ack, after that there will follow an unregister. - -```plantuml -@startjson -{ - "type": "stop", - "message_id" : "uuid", - "capability_id": "uuid" -} -@endjson -``` - -#### progress -Ask for the progress of the execution of the -```plantuml -@startjson +`commands` and `targets` are both plain arrays (0, 1, or many). SOARCA +never splits a single step across multiple jobs — one poll-able `Job` +always corresponds to exactly one step invocation, and it is entirely up to +the claiming Fin how to execute across however many targets it was given +(sequentially, or fanned out internally). An empty `targets` array is a +valid, spec-permitted shape: the Fin still runs `commands` once, without a +resolved target/authentication context, rather than SOARCA treating "no +targets" as "nothing to do." + +`targets[].target`/`targets[].authentication` reuse the same resolved +target/authentication shape used internally throughout SOARCA (and by the +Manual capability's API) — see +[`capability.ResolvedTarget`](https://github.com/COSSAS/SOARCA/blob/main/pkg/core/capability/capability.go). + +### Submitting a result + +Once a Fin has finished (or given up on) a job, it submits the result via +`PUT /fin/jobs/{job_id}`. Only the Fin the job is currently leased to may +submit a result for it — a valid `fin_token` alone is not sufficient to act +on another Fin's job. + +```json +PUT /fin/jobs/{job_id} { - "type": "progress", - "message_id" : "uuid", - "capability_id": "uuid" -} -@endjson -``` - -### Status response -|field |content |type | description | -| ----------------- | ------------- | ------ | ----------- | -|type |status |string |Message type -|message_id |UUID |string |message uuid -|capability_id |UUID |string |Capability uuid to control -|progress |ready, working, paused, stopped |string |Progress of the execution or state it's in. - -Report the progress of the execution of the capability - -```plantuml -@startjson -{ - "type": "status", - "message_id" : "uuid", - "capability_id": "uuid", - "progress": "" -} -@endjson -``` - -### Common -These contain command parts that are used in different messages. - -#### Security -|field |content |type | description | -| ----------------- | ------------- | ------ | ----------- | -|version |version |string |Version information of the protocol in [semantic version](https://semver.org) schema e.g. 1.2.4-beta -|channel_security |plaintext |string |Security mechanism used for encrypting the channel and topic, plaintext is only supported at this time - - -```plantuml -@startjson -{ - "security": { - "version": "0.0.0", - "channel_security": "plaintext" + "state": "success", + "variables": { + "__example__": { "type": "string", "value": "output" } } } -@endjson ``` -#### Variables -Variables information structure - -|field |content |type | description | -| ----------------- | ------------- | ------ | ----------- | -|type |variable type |[variable-type-ov](https://docs.oasis-open.org/cacao/security-playbooks/v2.0/cs01/security-playbooks-v2.0-cs01.html#_Toc152256556) | The cacao variable type see CACAO V2 chapter 10.18, 10.18.4 Variable Type Vocabulary -|name |name |string |Name of the variable this `must` be the same as the key on the map -|description |description |string |Description of the variable -|value |value |string |Value of the variable -|constant |true or false |bool |whether it is constant -|external |true or false |bool |whether it is external to the playbook - - -```plantuml -@startjson -{ - "____": { - "type": "", - "name": "", - "description": "", - "value": "", - "constant": "", - "external": "" - } -} -@endjson -``` - -#### Context -CACAO playbook context information structure - -|field |content |type | description | -| ----------------- | ------------- | ------ | ----------- | -|completed_on `(optional)` |timestamp |string | -|generated_on `(optional)` |timestamp |string | -|timeout `(optional)` |duration |string | -|step_id |UUID |string |Step uuid that is referred to -|playbook_id |UUID |string |Playbook uuid that is referred to -|execution_id |UUID |string |SOARCA execution uuid - -```plantuml -@startjson +`state` is `"success"` or `"failure"` and, together with `variables`, is +the only part of the result SOARCA's step machinery (`on_completion` +branching, downstream variable interpolation) actually reads — matching +CACAO's own model, which has no notion of per-target outcomes. If a Fin +processed multiple targets, it computes this single aggregated +success/failure using a fail-if-any policy, and a last-write-wins merge for +`variables`. + +An optional `target_results` array may additionally be included, giving +per-target diagnostic detail (which target, which command index failed, +per-target variables/error) — this is purely additive, for +reporting/audit/dashboards, and is never consulted by playbook control +flow. + +### Status pings (long-running jobs) + +For jobs that take more than a few seconds, a Fin should periodically call +`PATCH /fin/jobs/{job_id}/status`. This extends the job's lease (so it +isn't requeued for another Fin while still legitimately being worked on), +and gives SOARCA a place to piggyback a pending instruction — currently +only job cancellation, surfaced as `{"action": "cancel"}` — without needing +any inbound-facing channel on the Fin side. + +```json +PATCH /fin/jobs/{job_id}/status { - "context": { - "completed_on": "string: ", - "generated_on": "string: ", - "timeout": "string: ", - "step_id": "uuid", - "playbook_id": "uuid", - "execution_id": "uuid" - } + "progress": "connected, running command 2 of 3" } -@endjson ``` -#### Meta -Meta information for the fin protocol structure - -|field |content |type | description | -| ----------------- | ------------- | ------ | ----------- | -|timestamp |timestamp |string | -|sender_id |UUID |string |Step uuid that is referred to - - -```plantuml -@startjson +```json +200 OK { - "meta": { - "timestamp": "string: ", - "sender_id": "uuid" - } + "action": "" } -@endjson ``` -## Sequences - -### Registering a capability - -```plantuml -@startuml - -participant "SOARCA" as soarca -participant Capability as fin - -soarca -> soarca : create [soarca] topic +{{% alert title="Note" color="info" %}} +Job cancellation is specified but not yet implemented — `action` is always +empty today. The response shape exists so Fin implementations can start +checking it now. +{{% /alert %}} -fin -> fin : create [fin UUID] topic -soarca <- fin : [soarca] register -soarca --> fin : [fin UUID] ack +### Unregistering -@enduml -``` +`DELETE /fin/`, authenticated with that Fin's own `fin_token`, removes its +own registration. There is no `fin_id` in the path - it's inferred from the +token, since a Fin can only ever unregister itself. -### Sending command +An admin/dashboard client can additionally force-remove *any* Fin's +registration via `DELETE /fin/{fin_id}` (admin JWT, not fin_token) - useful +for cleaning up a stale/offline Fin that will never come back to +unregister itself. -```plantuml -@startuml +### Discovery -participant "SOARCA" as soarca -participant Capability as fin +`GET /fin/` and `GET /fin/{fin_id}` are ordinary, admin-JWT-gated reads (the +same authentication as the rest of SOARCA's admin API) for operators and +dashboards to see which Fins are registered, their declared capabilities, +and when they were last seen polling. `last_seen` is observability only — +a Fin that stops polling is not actively expired or hidden from routing; +jobs queued under its capability types simply go unclaimed until the +enqueuing step's own timeout elapses. -soarca -> fin : [capability UUID] command -soarca <-- fin : [capability UUID] ack +## Lease and retry semantics -.... processing .... - -soarca <- fin : [capability UUID] result -soarca --> fin: ack - -@enduml -``` - -### Unregistering a capability +Every job carries a lease (`lease_expires_in_seconds`, sized off the step's +own timeout). If the claiming Fin neither submits a result nor sends a +status ping before the lease expires, the job is automatically requeued for +any other Fin registered under the same capability type — this is the +mechanism that provides retry/failover across a pool without SOARCA needing +to detect a crashed or disconnected Fin explicitly. +## Sequence overview ```plantuml @startuml - participant "SOARCA" as soarca -participant Capability as fin -participant "Second capability" as fin2 - -... SOARCA initiate unregistering one fin ... +participant "Fin" as fin -soarca -> fin : [SOARCA] unregister fin-id -soarca <-- fin : [SOARCA] ack -note right fin2 - This capability does not respond to this message -end note +fin -> soarca : POST /fin/register (registration_token) +soarca --> fin : 201 (fin_id, fin_token) -... Fin initiate unregistering ... +loop poll loop + fin -> soarca : POST /fin/poll (fin_token) + soarca --> fin : 204 (no work) or 200 (job) +end -soarca <- fin : [SOARCA] unregister fin-id -soarca --> fin : [SOARCA] ack -note right fin2 - This capability does not respond to this message -end note +note over fin : job claimed, executing... -... SOARCA unregister all ... +opt long-running job + fin -> soarca : PATCH /fin/jobs/{job_id}/status + soarca --> fin : 200 (action, if any) +end -soarca -> fin : [SOARCA] unregister all == true -soarca <-- fin : [SOARCA] ack -soarca <-- fin2 : [SOARCA] ack -note over soarca, fin2 - soarca will go down after this command -end note +fin -> soarca : PUT /fin/jobs/{job_id} (result) +soarca --> fin : 204 @enduml ``` -### Control - -```plantuml -@startuml - -participant "SOARCA" as soarca -participant Capability as fin - - -soarca -> fin : [fin UUID] control message -soarca <-- fin : [fin UUID] status - -@enduml -``` - - +## Example playbook +See [`examples/fin-playbook.json`](https://github.com/COSSAS/SOARCA/blob/main/examples/fin-playbook.json) +for a worked example combining a native (SSH) capability step with a step +targeting a registered Fin capability type. diff --git a/docs/content/en/docs/soarca-extensions/fin-setup-and-python-library.md b/docs/content/en/docs/soarca-extensions/fin-setup-and-python-library.md deleted file mode 100644 index 8db6d8e4..00000000 --- a/docs/content/en/docs/soarca-extensions/fin-setup-and-python-library.md +++ /dev/null @@ -1,262 +0,0 @@ ---- -title: Fin Setup and Python Library -description: > - Documentation of the Python Fin library -categories: [extensions, architecture] -tags: [fin, python] -weight: 2 -date: 2024-04-10 ---- - -## Quick Start - deployment - -To deploy SOARCA fins a few things need to happen: - -1. Have SOARCA deployed and up -2. Have a Fin developed -3. Ability to contact SOARCA API from the Fin location - -To deploy the example use: - -```bash -git clone git@github.com:COSSAS/SOARCA-FIN-python-library.git -cd ~/examples -pip install -r requirements.txt -python3 pong_example.py -``` -Now the example playbook can be from the SOARCA main repo - -## Quick Start - development - -For the documentation about the Fin protocol we refer to documention page of [SOARCA Fin Protocol](https://cossas.github.io/SOARCA/docs/soarca-extensions/fin-protocol/). - -To include the SOARCA Fin library, you can use the following command to install it via pip: - -```bash -pip install soarca-fin-library -``` - -### Example -An example on how to use the library is given below. -For more examples and the source code, we will refer to the Github page of the [SOARCA-Fin-python-library](https://github.com/COSSAS/SOARCA-FIN-python-library), where we provide `/examples` folder. - -```python -import os -from dotenv import load_dotenv - -from soarca_fin_python_library.soarca_fin import SoarcaFin -from soarca_fin_python_library.models.agent_structure import AgentStructure -from soarca_fin_python_library.models.external_reference import ExternalReference -from soarca_fin_python_library.models.step_structure import StepStructure -from soarca_fin_python_library.models.capability_structure import CapabilityStructure -from soarca_fin_python_library.enums.workflow_step_enum import WorkFlowStepEnum -from soarca_fin_python_library.models.command import Command -from soarca_fin_python_library.models.result_structure import ResultStructure - -from soarca_fin_python_library.models.variable import Variable -from soarca_fin_python_library.enums.variable_type_enum import VariableTypeEnum - - -def capability_pong_callback(command: Command) -> ResultStructure: - print("Received ping, returning pong!") - - result = Variable( - type=VariableTypeEnum.string, - name="pong_output", - description="If ping, return pong", - value="pong", - constant=True, - external=False) - - context = command.command.context - - return ResultStructure( - state="success", context=context, variables={"result": result}) - - -def main(mqtt_broker: str, mqtt_port: int, username: str, password: str) -> None: - - finId = "soarca-fin--pingpong-f877bb3a-bb37-429e-8ece-2d4286cf326d" - agentName = "soarca-fin-pong-f896bb3b-bb37-429e-8ece-2d4286cf326d" - externalReferenceName = "external-reference-example-name" - capabilityId = "mod-pong--e896aa3b-bb37-429e-8ece-2d4286cf326d" - - # Create AgentStructure - agent = AgentStructure( - name=agentName) - - # Create ExternalReference - external_reference = ExternalReference(name=externalReferenceName) - - # Create StepStructure - step_structure = StepStructure( - name="step_name", - description="step description", - external_references=[external_reference], - command="pong", - target=agentName) - - # Create CapabilityStructure - capability_structure = CapabilityStructure( - capability_id=capabilityId, - type=WorkFlowStepEnum.action, - name="Ping Pong capability", - version="0.0.1", - step={ - "test": step_structure}, - agent={ - "testagent": agent}) - - # Create Soarca fin - fin = SoarcaFin(finId) - # Set config for MQTT Server - fin.set_config_MQTT_server(mqtt_broker, mqtt_port, username, password) - # Register Capabilities - fin.create_fin_capability(capability_structure, capability_pong_callback) - # Start the fin - fin.start_fin() - - -if __name__ == "__main__": - load_dotenv() - MQTT_BROKER = os.getenv("MQTT_BROKER", "localhost") - MQTT_PORT = int(os.getenv("MQTT_PORT", "1883")) - USERNAME = os.getenv("MQTT_USERNAME", "soarca") - PASSWD = os.getenv("MQTT_PASSWD", "password") - - main(MQTT_BROKER, MQTT_PORT, USERNAME, PASSWD) - -``` - -Below we have provided an example env file. Note that this changes according to your setup. - -``` -MQTT_BROKER = "localhost" -MQTT_PORT = "1883" -MQTT_USERNAME = "soarca" -MQTT_PASSWD = "password" -``` - -Env file can be exported by running: -```bash -export $(cat .env | grep -v "#" | xargs) -``` - -## Architecture -The main object of the application is the `SoarcaFin` object, which is responsible for configuring and creating and controlling the capabilities. -The SoarcaFin creates `MQTTClient`s for each capability registered, plus one for registering, unregistering and controlling the fi itself. -`MQTTClient`s each have their own connection to the MQTT Broker and own `Parser` and `Executor` objects. -The `Parser` object parsers the raw MQTT messages and tries to convert them to one of the objects in `src/models`. -The `Executor` runs in their own thread and handles the actual execution of the messages. -The `Executor` polls a thread-safe queue for new messages and performs IO operations, such as sending messages to the MQTT broker and calling capability callbacks. - -### Setup SOARCA Capabilities - - -To register a fin to SOARCA, first create a `SoarcaFin` object and pass the `fin_id` in the constructor. The SOARCA `fin_id` must be in the format of: `sourca-fin--`. -Call `set_config_MQTT_server()` to set the required configurations for the fin to connect to the MQTT broker. -For each capability to be registered, call `create_fin_capability()`. The capability callback funtion should return an object of type `ResultStructure`. -When all capabilities are initialized, call `start_fin()` for the SOARCA Fin to connect to the MQTT broker and register itself to SOARCA. - -An example is given in this project in the file [`examples/pong_example.py`] - -### Class Overview -```plantuml -interface IParser { - Message parse_on_message() -} - -interface IMQTTClient { - void on_connect() - void on_message() -} - -interface ISoarcaFin { - void set_config_MQTTServer() - void set_fin_capabilities() - void start_fin() -} - -interface IExecutor { - void queue_message() -} - - -class SoarcaFin -class MQTTClient -class Parser -class Executor - -ISoarcaFin <|.. SoarcaFin -IMQTTClient <|.. MQTTClient -IParser <|.. Parser -IExecutor <|.. Executor - -IMQTTClient <- SoarcaFin -MQTTClient -> IExecutor -IParser <-MQTTClient -``` - -### Sequence Diagrams -#### Command -```plantuml -Soarca -> "MQTTClient (Capability 1)" : Command Message [Capability ID Topic] - -"MQTTClient (Capability 1)" -> Parser : parse_on_message(message) -"MQTTClient (Capability 1)" <-- Parser : Message.Command - -"MQTTClient (Capability 1)" -> "Executor (Capability 1)" : Command message -Soarca <-- "Executor (Capability 1)" : Ack - -"Executor (Capability 1)" -> "Capability Callback" : Command -"Executor (Capability 1)" <-- "Capability Callback" : Result - - -Soarca <- "Executor (Capability 1)" : Result -Soarca --> "MQTTClient (Capability 1)" : Ack - -"MQTTClient (Capability 1)" -> Parser : parse_on_message(message) -"MQTTClient (Capability 1)" <-- Parser : Message.Ack - -"MQTTClient (Capability 1)" -> "Executor (Capability 1)" : Ack message -``` - -#### Register -```plantuml -Soarca -> Soarca : Create Soarca Topic - -Library -> SoarcaFin : Set MQTT Server config - -Library -> SoarcaFin : Set Capability1 -SoarcaFin -> "MQTTClient (Capability 1)" : Create capability - -Library -> SoarcaFin : Set Capability2 -SoarcaFin -> "MQTTClient (Capability 2)" : Create capability - - -Library -> SoarcaFin : Start Fin - - -SoarcaFin -> "MQTTClient (Capability 1)" : Start capability -"MQTTClient (Capability 1)" -> "MQTTClient (Capability 1)" : Register Capability Topic -SoarcaFin -> "MQTTClient (Capability 2)" : Start capability -"MQTTClient (Capability 2)" -> "MQTTClient (Capability 2)" : Register Capability Topic - -SoarcaFin -> "MQTTClient (Fin)" : Register Fin -"MQTTClient (Fin)" -> "MQTTClient (Fin)" : Register SoarcaFin Topic - -"MQTTClient (Fin)" -> "Executor (Fin)" : Send Register Message - -Soarca <- "Executor (Fin)" : Message.Register [Soarca Topic] - -Soarca --> "MQTTClient (Fin)" : Message.Ack [Fin ID Topic] - -"MQTTClient (Fin)" -> "Parser (Fin)" : parse_on_message(ack) -"MQTTClient (Fin)" <-- "Parser (Fin)" : Message.Ack - -"MQTTClient (Fin)" -> "Executor (Fin)" : Message.Ack -``` - -## Bugs or Contributing -Want to contribute to this project? It is possible to contribute [here](https://github.com/COSSAS/SOARCA-FIN-python-library). -Have you found a bug or want to request a feature? Please create an issue [here](https://github.com/COSSAS/SOARCA-FIN-python-library/issues). \ No newline at end of file diff --git a/docs/content/en/docs/soarca-extensions/native-capabilities.md b/docs/content/en/docs/soarca-extensions/native-capabilities.md index 8a15a09a..e22e64f0 100644 --- a/docs/content/en/docs/soarca-extensions/native-capabilities.md +++ b/docs/content/en/docs/soarca-extensions/native-capabilities.md @@ -8,7 +8,7 @@ weight: 2 date: 2023-01-05 --- -This page contains a list of capabilities that are natively implemented in SOARCA see details [here](/docs/core-components/modules). For MQTT-message-based capabilities, check [here](/docs/soarca-extensions/). +This page contains a list of capabilities that are natively implemented in SOARCA see details [here](/docs/core-components/modules). For Fin-based extension capabilities, check [here](/docs/soarca-extensions/). ## OpenC2 capability diff --git a/docs/static/openapi/swagger.json b/docs/static/openapi/swagger.json index f14e4f60..0fcdd272 100644 --- a/docs/static/openapi/swagger.json +++ b/docs/static/openapi/swagger.json @@ -6,28 +6,40 @@ "version": "1.0.0" }, "paths": { - "/keymanagement/": { + "/fin/": { "get": { - "description": "return all keys in the KMS", + "description": "list all currently-registered fins and their capabilities", "produces": [ "application/json" ], "tags": [ - "keymanagement" + "fin" ], - "summary": "gets all keys from the KMS", + "summary": "list all currently-registered fins and their capabilities", "responses": { "200": { "description": "OK", "schema": { - "type": "array", - "items": { - "type": "string" - } + "$ref": "#/definitions/fin.ListResponse" } + } + } + }, + "delete": { + "description": "delete this Fin's own registration. The fin_id is inferred from the fin_token presented in the Authorization header - a Fin can only ever delete its own registration, so it never needs to name itself explicitly.", + "produces": [ + "application/json" + ], + "tags": [ + "fin" + ], + "summary": "delete this Fin's own registration", + "responses": { + "204": { + "description": "No Content" }, - "400": { - "description": "Bad Request", + "404": { + "description": "Not Found", "schema": { "$ref": "#/definitions/api.Error" } @@ -35,24 +47,89 @@ } } }, - "/keymanagement/:keyname/": { + "/fin/jobs/{job_id}": { "put": { - "description": "adds a key to the KMS; load key into cache and write file", + "description": "submit the result of a claimed job. Only the Fin the job is currently leased to may submit a result for it.", + "consumes": [ + "application/json" + ], "produces": [ "application/json" ], "tags": [ - "keymanagement" + "fin" ], - "summary": "add key to KMS", + "summary": "submit the result of a claimed job", "parameters": [ { - "description": "key", + "type": "string", + "description": "job ID", + "name": "job_id", + "in": "path", + "required": true + }, + { + "description": "job result", "name": "data", "in": "body", "required": true, "schema": { - "$ref": "#/definitions/api.KeyManagementKey" + "$ref": "#/definitions/fin.ResultRequest" + } + } + ], + "responses": { + "204": { + "description": "No Content" + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/api.Error" + } + } + } + } + }, + "/fin/jobs/{job_id}/status": { + "patch": { + "description": "extend a claimed job's lease (so a legitimately long-running job isn't requeued out from under the Fin still working on it), and check for a pending instruction such as cancellation", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "fin" + ], + "summary": "extend a claimed job's lease, and check for a pending cancellation instruction", + "parameters": [ + { + "type": "string", + "description": "job ID", + "name": "job_id", + "in": "path", + "required": true + }, + { + "description": "progress", + "name": "data", + "in": "body", + "schema": { + "$ref": "#/definitions/fin.StatusPingRequest" } } ], @@ -60,7 +137,7 @@ "200": { "description": "OK", "schema": { - "type": "json" + "$ref": "#/definitions/fin.StatusPingResponse" } }, "400": { @@ -68,23 +145,93 @@ "schema": { "$ref": "#/definitions/api.Error" } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/api.Error" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/api.Error" + } } } - }, - "delete": { - "description": "revokes the key by moving it to .revoked and renaming it", + } + }, + "/fin/poll": { + "post": { + "description": "long-poll for the next job matching this Fin's registered capability types. Returns 204 No Content if long_poll_timeout_seconds elapses with no job available - callers should simply poll again.", + "consumes": [ + "application/json" + ], "produces": [ "application/json" ], "tags": [ - "keymanagement" + "fin" + ], + "summary": "long-poll for the next job matching this Fin's registered capability types", + "parameters": [ + { + "description": "poll hints", + "name": "data", + "in": "body", + "schema": { + "$ref": "#/definitions/fin.PollRequest" + } + } ], - "summary": "remove key from KMS", "responses": { "200": { "description": "OK", "schema": { - "type": "json" + "$ref": "#/definitions/fin.PollResponse" + } + }, + "204": { + "description": "No Content" + }, + "401": { + "description": "Unauthorized", + "schema": { + "$ref": "#/definitions/api.Error" + } + } + } + } + }, + "/fin/register": { + "post": { + "description": "register a new Fin process, declaring the capability types it can execute, and obtain its fin_id/fin_token", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "fin" + ], + "summary": "register a new Fin and obtain its fin_token", + "parameters": [ + { + "description": "registration", + "name": "data", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/fin.RegisterRequest" + } + } + ], + "responses": { + "201": { + "description": "Created", + "schema": { + "$ref": "#/definitions/fin.RegisterResponse" } }, "400": { @@ -92,29 +239,74 @@ "schema": { "$ref": "#/definitions/api.Error" } + }, + "403": { + "description": "Forbidden", + "schema": { + "$ref": "#/definitions/api.Error" + } } } } }, - "/keymanagement/refresh/": { - "post": { - "description": "refresh the KMS and re-parse the underlying directory", + "/fin/{fin_id}": { + "get": { + "description": "look up a specific registered fin by id", "produces": [ "application/json" ], "tags": [ - "keymanagement" + "fin" + ], + "summary": "look up a specific registered fin by id", + "parameters": [ + { + "type": "string", + "description": "fin ID", + "name": "fin_id", + "in": "path", + "required": true + } ], - "summary": "refresh the KMS system, which allows the user to include manually added files in the KMS", "responses": { "200": { "description": "OK", "schema": { - "type": "json" + "$ref": "#/definitions/fin.Record" } }, - "400": { - "description": "Bad Request", + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/api.Error" + } + } + } + }, + "delete": { + "description": "forcibly remove a registered fin's record, e.g. one that is stale/offline and will never come back to unregister itself. This is an admin/dashboard action, not Fin-authenticated - unlike Unregister, it is not restricted to a fin removing its own registration.", + "produces": [ + "application/json" + ], + "tags": [ + "fin" + ], + "summary": "forcibly remove a registered fin (admin)", + "parameters": [ + { + "type": "string", + "description": "fin ID", + "name": "fin_id", + "in": "path", + "required": true + } + ], + "responses": { + "204": { + "description": "No Content" + }, + "404": { + "description": "Not Found", "schema": { "$ref": "#/definitions/api.Error" } @@ -157,9 +349,9 @@ } } }, - "/manual/continue": { - "post": { - "description": "updates the value of a variable according to the manual interaction", + "/manual/{exec_id}/{step_execution_id}": { + "get": { + "description": "get a specific manual command that still needs a value to be returned", "consumes": [ "application/json" ], @@ -169,7 +361,7 @@ "tags": [ "manual" ], - "summary": "updates the value of a variable according to the manual interaction", + "summary": "get a specific manual command that still needs a value to be returned", "parameters": [ { "type": "string", @@ -180,26 +372,17 @@ }, { "type": "string", - "description": "step ID", - "name": "step_id", + "description": "step execution ID (identifies a specific pending step invocation; see GET /manual/ to discover it, as multiple pending commands may share the same step ID)", + "name": "step_execution_id", "in": "path", "required": true - }, - { - "description": "playbook", - "name": "data", - "in": "body", - "required": true, - "schema": { - "$ref": "#/definitions/api.ManualOutArgsUpdatePayload" - } } ], "responses": { "200": { "description": "OK", "schema": { - "$ref": "#/definitions/api.Execution" + "$ref": "#/definitions/api.InteractionCommandData" } }, "400": { @@ -209,11 +392,9 @@ } } } - } - }, - "/manual/{exec_id}/{step_id}": { - "get": { - "description": "get a specific manual command that still needs a value to be returned", + }, + "put": { + "description": "resolve a specific pending manual command by supplying its out args. This is a PUT\non the same resource GET /manual/{exec_id}/{step_execution_id} identifies, not a\ngeneric RPC-style action, so the ids live in the path, not the body.", "consumes": [ "application/json" ], @@ -223,7 +404,7 @@ "tags": [ "manual" ], - "summary": "get a specific manual command that still needs a value to be returned", + "summary": "resolve a specific pending manual command by supplying its out args", "parameters": [ { "type": "string", @@ -234,17 +415,26 @@ }, { "type": "string", - "description": "step ID", - "name": "step_id", + "description": "step execution ID (identifies a specific pending step invocation; see GET /manual/ to discover it, as multiple pending commands may share the same step ID)", + "name": "step_execution_id", "in": "path", "required": true + }, + { + "description": "resolution", + "name": "data", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/api.ManualOutArgsUpdatePayload" + } } ], "responses": { "200": { "description": "OK", "schema": { - "$ref": "#/definitions/api.InteractionCommandData" + "$ref": "#/definitions/api.Execution" } }, "400": { @@ -713,27 +903,22 @@ "api.InteractionCommandData": { "type": "object", "required": [ - "command", - "description", + "commands", "execution_id", "out_args", "playbook_id", + "step_execution_id", "step_id", - "target", + "targets", "type" ], "properties": { - "command": { - "description": "The command for the agent either command", - "type": "string" - }, - "commandb64": { - "description": "Indicates if the command is in b64", - "type": "boolean" - }, - "description": { - "description": "The description from the workflow step", - "type": "string" + "commands": { + "description": "All commands of the step, in order. A manual step is a single unit of work resolved by one response, but may list multiple commands/instructions", + "type": "array", + "items": { + "$ref": "#/definitions/api.ManualCommand" + } }, "execution_id": { "description": "The id of the execution", @@ -751,17 +936,20 @@ "description": "The id of the CACAO playbook executed by the execution", "type": "string" }, + "step_execution_id": { + "description": "The id of this specific step invocation. Distinguishes concurrent/repeated pending commands that share the same StepId (e.g. overlapping loop iterations)", + "type": "string" + }, "step_id": { "description": "The id of the step executed by the execution", "type": "string" }, - "target": { - "description": "Map of cacao agent-target with the target(s) of this command", - "allOf": [ - { - "$ref": "#/definitions/cacao.AgentTarget" - } - ] + "targets": { + "description": "All targets of the step, in order, together with their resolved authentication information (needed by a human operator to perform the step manually)", + "type": "array", + "items": { + "$ref": "#/definitions/capability.ResolvedTarget" + } }, "type": { "description": "The type of this content", @@ -770,13 +958,23 @@ } } }, - "api.KeyManagementKey": { + "api.ManualCommand": { "type": "object", + "required": [ + "command", + "description" + ], "properties": { - "private": { + "command": { + "description": "The command for the agent, either plain or base64", "type": "string" }, - "public": { + "commandb64": { + "description": "Indicates if the command is in b64", + "type": "boolean" + }, + "description": { + "description": "The description from the workflow step", "type": "string" } } @@ -784,22 +982,11 @@ "api.ManualOutArgsUpdatePayload": { "type": "object", "required": [ - "execution_id", - "playbook_id", "response_out_args", "response_status", - "step_id", "type" ], "properties": { - "execution_id": { - "description": "The id of the execution", - "type": "string" - }, - "playbook_id": { - "description": "The id of the CACAO playbook executed by the execution", - "type": "string" - }, "response_out_args": { "description": "Map of cacao variables storing the out args value, handled in the step out args, with current values and definitions", "allOf": [ @@ -816,10 +1003,6 @@ } ] }, - "step_id": { - "description": "The id of the step executed by the execution", - "type": "string" - }, "type": { "description": "The type of this content", "type": "string", @@ -950,6 +1133,9 @@ "status_text": { "type": "string" }, + "step_execution_id": { + "type": "string" + }, "step_id": { "type": "string" }, @@ -1729,6 +1915,293 @@ "$ref": "#/definitions/cacao.Step" } }, + "capability.ResolvedTarget": { + "type": "object", + "required": [ + "target" + ], + "properties": { + "authentication": { + "$ref": "#/definitions/cacao.AuthenticationInformation" + }, + "target": { + "$ref": "#/definitions/cacao.AgentTarget" + } + } + }, + "fin.Command": { + "type": "object", + "properties": { + "command": { + "type": "string" + }, + "command_b64": { + "type": "string" + }, + "content": { + "type": "string" + }, + "content_b64": { + "type": "string" + }, + "headers": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "type": { + "type": "string" + } + } + }, + "fin.Job": { + "type": "object", + "properties": { + "capability_type": { + "description": "CapabilityType is the routing key this job was queued under — it\nmatches Capability.Type of whichever Fin ultimately claims it.", + "type": "string" + }, + "commands": { + "type": "array", + "items": { + "$ref": "#/definitions/fin.Command" + } + }, + "execution_id": { + "description": "ExecutionId/PlaybookId/StepId/StepExecutionId identify which\nexecution/playbook/step invocation this job belongs to.\nStepExecutionId disambiguates concurrent/repeated invocations of the\nsame StepId within one execution (e.g. while-loop iterations),\nmatching how the rest of the execution model is keyed.", + "type": "string" + }, + "job_id": { + "description": "JobId identifies this specific poll-able unit of work (== the lease\nhandle used for result submission and status pings).", + "type": "string" + }, + "lease_expires_in_seconds": { + "description": "LeaseExpiresInSeconds mirrors a queue visibility timeout: sized off\nthe step's own timeout, and expected to cover however many targets\nthe claiming Fin ends up processing. If no result or status ping\narrives before it elapses, the job is requeued for any other Fin\nregistered under the same CapabilityType.", + "type": "integer" + }, + "playbook_id": { + "type": "string" + }, + "step": { + "$ref": "#/definitions/fin.StepInfo" + }, + "step_execution_id": { + "type": "string" + }, + "step_id": { + "type": "string" + }, + "targets": { + "type": "array", + "items": { + "$ref": "#/definitions/capability.ResolvedTarget" + } + }, + "variables": { + "$ref": "#/definitions/cacao.Variables" + } + } + }, + "fin.JobState": { + "type": "string", + "enum": [ + "success", + "failure" + ], + "x-enum-varnames": [ + "JobStateSuccess", + "JobStateFailure" + ] + }, + "fin.ListResponse": { + "type": "object", + "properties": { + "fins": { + "type": "array", + "items": { + "$ref": "#/definitions/fin.Record" + } + } + } + }, + "fin.PollRequest": { + "type": "object", + "properties": { + "concurrency_available": { + "description": "ConcurrencyAvailable is an optional hint: how many more jobs this Fin\ncould take on right now. Lets SOARCA prefer idle Fins in a capability\npool over busy ones. A Fin that only ever runs one job at a time can\nomit it or always send 1/0.", + "type": "integer" + } + } + }, + "fin.PollResponse": { + "type": "object", + "properties": { + "job": { + "$ref": "#/definitions/fin.Job" + } + } + }, + "fin.Record": { + "type": "object", + "properties": { + "capabilities": { + "type": "array", + "items": { + "$ref": "#/definitions/pkg_models_fin.Capability" + } + }, + "display_name": { + "description": "DisplayName is free text for humans/logs only; it plays no role in\nrouting or identity.", + "type": "string" + }, + "fin_id": { + "description": "FinId is server-assigned at registration (never client-chosen),\navoiding id collisions and any implicit trust that a Fin picks a\nunique id for itself.", + "type": "string" + }, + "last_seen": { + "description": "LastSeen is updated on every /poll call (whether or not a job was\nreturned), every job result submission, and every status ping (the\nheartbeat for a long-running job, §2.5) — so a Fin busy working a\njob via status pings is not wrongly treated as gone dark just\nbecause it isn't polling. Surfaced via List/Get for operator/\ndashboard visibility into which registered Fins are actually still\nalive, and used to derive Stale below. A Fin that stops all of the\nabove is never actively\nexpired or removed from routing (see\ndocs/adr/FIN-WEBHOOK-PROTOCOL-PROPOSAL.md §2.4): its registration\nstays in place, but pkg/core/capability/fin.Capability's fail-fast\nliveness check uses LastSeen to decide whether a step should even\nbother enqueuing a job for it - once every Fin declaring a given\ncapability type is stale, new jobs of that type are failed\nimmediately instead of being enqueued to wait out the step's own\ntimeout unclaimed.", + "type": "string" + }, + "protocol_version": { + "type": "string" + }, + "registered_at": { + "type": "string" + }, + "stale": { + "description": "Stale is computed (never persisted - bson:\"-\") by pkg/api/fin's\nList/Get handlers, using the same staleness threshold as\npkg/core/capability/fin.Capability's fail-fast liveness check: true\nonce this Fin hasn't been seen (via /poll) in longer than that\nthreshold, so a dashboard can flag it as likely no longer running\nwithout SOARCA having to actively expire or hide the registration\nitself.", + "type": "boolean" + } + } + }, + "fin.RegisterRequest": { + "type": "object", + "required": [ + "capabilities", + "registration_token" + ], + "properties": { + "capabilities": { + "type": "array", + "items": { + "$ref": "#/definitions/pkg_models_fin.Capability" + } + }, + "display_name": { + "type": "string" + }, + "protocol_version": { + "type": "string" + }, + "registration_token": { + "description": "RegistrationToken gates *creating* a new Fin identity. It is a\ncoarse, admin/instance-level shared secret (configured server-side as\nFIN_REGISTRATION_TOKEN, distributed out-of-band), sent as a plain\nbody field because registration is the one call that happens before\na Fin has any credential of its own to put in a header.", + "type": "string" + } + } + }, + "fin.RegisterResponse": { + "type": "object", + "properties": { + "fin_id": { + "type": "string" + }, + "fin_token": { + "type": "string" + }, + "job_lease_seconds": { + "type": "integer" + }, + "long_poll_timeout_seconds": { + "type": "integer" + }, + "poll_interval_seconds": { + "description": "PollIntervalSeconds/LongPollTimeoutSeconds/JobLeaseSeconds are\nserver-chosen operational defaults, echoed back so a Fin doesn't need\nits own hardcoded copy of them.", + "type": "integer" + } + } + }, + "fin.ResultRequest": { + "type": "object", + "required": [ + "state" + ], + "properties": { + "error": { + "type": "string" + }, + "state": { + "$ref": "#/definitions/fin.JobState" + }, + "target_results": { + "type": "array", + "items": { + "$ref": "#/definitions/fin.TargetResult" + } + }, + "variables": { + "$ref": "#/definitions/cacao.Variables" + } + } + }, + "fin.StatusPingRequest": { + "type": "object", + "properties": { + "progress": { + "type": "string" + } + } + }, + "fin.StatusPingResponse": { + "type": "object", + "properties": { + "action": { + "type": "string" + } + } + }, + "fin.StepInfo": { + "type": "object", + "properties": { + "delay": { + "type": "integer" + }, + "description": { + "type": "string" + }, + "name": { + "type": "string" + }, + "timeout": { + "type": "integer" + } + } + }, + "fin.TargetResult": { + "type": "object", + "properties": { + "error": { + "type": "string" + }, + "failed_command_index": { + "description": "FailedCommandIndex identifies which command in Commands aborted this\ntarget's sequence, if any.", + "type": "integer" + }, + "state": { + "$ref": "#/definitions/fin.JobState" + }, + "target_index": { + "description": "TargetIndex identifies which entry in the job's Targets this result\nis for. Nil when the job had no targets at all.", + "type": "integer" + }, + "variables": { + "$ref": "#/definitions/cacao.Variables" + } + } + }, "manual.ManualResponseStatus": { "type": "string", "enum": [ @@ -1739,6 +2212,33 @@ "ManualResponseSuccessStatus", "ManualResponseFailureStatus" ] + }, + "pkg_models_fin.Capability": { + "type": "object", + "required": [ + "type" + ], + "properties": { + "description": { + "description": "Description is free text for humans (dashboards, logs) — it plays no\nrole in routing.", + "type": "string" + }, + "step_examples": { + "description": "StepExamples are optional, illustrative CACAO action steps showing\nhow a playbook author would invoke this capability (agent, commands,\ntargets, ...), surfaced to help playbook authors — they are never\ninterpreted or validated by SOARCA. A capability may offer more than\none example (e.g. to illustrate different commands or target shapes).", + "type": "array", + "items": { + "$ref": "#/definitions/cacao.Step" + } + }, + "type": { + "description": "Type is the routing key: the value playbook authors write into\nagent_definitions[...].type to target this capability pool. Multiple\nindependently-deployed Fin processes may register the same Type;\nSOARCA treats them as one interchangeable pool and load-balances/\nfails over across them (see docs/adr/FIN-WEBHOOK-PROTOCOL-PROPOSAL.md\n§2.1b).", + "type": "string" + }, + "version": { + "description": "Version is the capability implementation's own version string, for\noperator/debugging visibility only.", + "type": "string" + } + } } } } \ No newline at end of file diff --git a/docs/static/openapi/swagger.yaml b/docs/static/openapi/swagger.yaml index b590e453..b7633819 100644 --- a/docs/static/openapi/swagger.yaml +++ b/docs/static/openapi/swagger.yaml @@ -32,15 +32,12 @@ definitions: type: object api.InteractionCommandData: properties: - command: - description: The command for the agent either command - type: string - commandb64: - description: Indicates if the command is in b64 - type: boolean - description: - description: The description from the workflow step - type: string + commands: + description: All commands of the step, in order. A manual step is a single + unit of work resolved by one response, but may list multiple commands/instructions + items: + $ref: '#/definitions/api.ManualCommand' + type: array execution_id: description: The id of the execution type: string @@ -52,42 +49,51 @@ definitions: playbook_id: description: The id of the CACAO playbook executed by the execution type: string + step_execution_id: + description: The id of this specific step invocation. Distinguishes concurrent/repeated + pending commands that share the same StepId (e.g. overlapping loop iterations) + type: string step_id: description: The id of the step executed by the execution type: string - target: - allOf: - - $ref: '#/definitions/cacao.AgentTarget' - description: Map of cacao agent-target with the target(s) of this command + targets: + description: All targets of the step, in order, together with their resolved + authentication information (needed by a human operator to perform the step + manually) + items: + $ref: '#/definitions/capability.ResolvedTarget' + type: array type: description: The type of this content example: execution-status type: string required: - - command - - description + - commands - execution_id - out_args - playbook_id + - step_execution_id - step_id - - target + - targets - type type: object - api.KeyManagementKey: + api.ManualCommand: properties: - private: + command: + description: The command for the agent, either plain or base64 type: string - public: + commandb64: + description: Indicates if the command is in b64 + type: boolean + description: + description: The description from the workflow step type: string + required: + - command + - description type: object api.ManualOutArgsUpdatePayload: properties: - execution_id: - description: The id of the execution - type: string - playbook_id: - description: The id of the CACAO playbook executed by the execution - type: string response_out_args: allOf: - $ref: '#/definitions/cacao.Variables' @@ -97,19 +103,13 @@ definitions: allOf: - $ref: '#/definitions/manual.ManualResponseStatus' description: Indicates status of command - step_id: - description: The id of the step executed by the execution - type: string type: description: The type of this content example: string type: string required: - - execution_id - - playbook_id - response_out_args - response_status - - step_id - type type: object api.PlaybookExecutionReport: @@ -193,6 +193,8 @@ definitions: type: string status_text: type: string + step_execution_id: + type: string step_id: type: string variables: @@ -724,6 +726,256 @@ definitions: additionalProperties: $ref: '#/definitions/cacao.Step' type: object + capability.ResolvedTarget: + properties: + authentication: + $ref: '#/definitions/cacao.AuthenticationInformation' + target: + $ref: '#/definitions/cacao.AgentTarget' + required: + - target + type: object + fin.Command: + properties: + command: + type: string + command_b64: + type: string + content: + type: string + content_b64: + type: string + headers: + additionalProperties: + items: + type: string + type: array + type: object + type: + type: string + type: object + fin.Job: + properties: + capability_type: + description: |- + CapabilityType is the routing key this job was queued under — it + matches Capability.Type of whichever Fin ultimately claims it. + type: string + commands: + items: + $ref: '#/definitions/fin.Command' + type: array + execution_id: + description: |- + ExecutionId/PlaybookId/StepId/StepExecutionId identify which + execution/playbook/step invocation this job belongs to. + StepExecutionId disambiguates concurrent/repeated invocations of the + same StepId within one execution (e.g. while-loop iterations), + matching how the rest of the execution model is keyed. + type: string + job_id: + description: |- + JobId identifies this specific poll-able unit of work (== the lease + handle used for result submission and status pings). + type: string + lease_expires_in_seconds: + description: |- + LeaseExpiresInSeconds mirrors a queue visibility timeout: sized off + the step's own timeout, and expected to cover however many targets + the claiming Fin ends up processing. If no result or status ping + arrives before it elapses, the job is requeued for any other Fin + registered under the same CapabilityType. + type: integer + playbook_id: + type: string + step: + $ref: '#/definitions/fin.StepInfo' + step_execution_id: + type: string + step_id: + type: string + targets: + items: + $ref: '#/definitions/capability.ResolvedTarget' + type: array + variables: + $ref: '#/definitions/cacao.Variables' + type: object + fin.JobState: + enum: + - success + - failure + type: string + x-enum-varnames: + - JobStateSuccess + - JobStateFailure + fin.ListResponse: + properties: + fins: + items: + $ref: '#/definitions/fin.Record' + type: array + type: object + fin.PollRequest: + properties: + concurrency_available: + description: |- + ConcurrencyAvailable is an optional hint: how many more jobs this Fin + could take on right now. Lets SOARCA prefer idle Fins in a capability + pool over busy ones. A Fin that only ever runs one job at a time can + omit it or always send 1/0. + type: integer + type: object + fin.PollResponse: + properties: + job: + $ref: '#/definitions/fin.Job' + type: object + fin.Record: + properties: + capabilities: + items: + $ref: '#/definitions/pkg_models_fin.Capability' + type: array + display_name: + description: |- + DisplayName is free text for humans/logs only; it plays no role in + routing or identity. + type: string + fin_id: + description: |- + FinId is server-assigned at registration (never client-chosen), + avoiding id collisions and any implicit trust that a Fin picks a + unique id for itself. + type: string + last_seen: + description: |- + LastSeen is updated on every /poll call (whether or not a job was + returned), every job result submission, and every status ping (the + heartbeat for a long-running job, §2.5) — so a Fin busy working a + job via status pings is not wrongly treated as gone dark just + because it isn't polling. Surfaced via List/Get for operator/ + dashboard visibility into which registered Fins are actually still + alive, and used to derive Stale below. A Fin that stops all of the + above is never actively + expired or removed from routing (see + docs/adr/FIN-WEBHOOK-PROTOCOL-PROPOSAL.md §2.4): its registration + stays in place, but pkg/core/capability/fin.Capability's fail-fast + liveness check uses LastSeen to decide whether a step should even + bother enqueuing a job for it - once every Fin declaring a given + capability type is stale, new jobs of that type are failed + immediately instead of being enqueued to wait out the step's own + timeout unclaimed. + type: string + protocol_version: + type: string + registered_at: + type: string + stale: + description: |- + Stale is computed (never persisted - bson:"-") by pkg/api/fin's + List/Get handlers, using the same staleness threshold as + pkg/core/capability/fin.Capability's fail-fast liveness check: true + once this Fin hasn't been seen (via /poll) in longer than that + threshold, so a dashboard can flag it as likely no longer running + without SOARCA having to actively expire or hide the registration + itself. + type: boolean + type: object + fin.RegisterRequest: + properties: + capabilities: + items: + $ref: '#/definitions/pkg_models_fin.Capability' + type: array + display_name: + type: string + protocol_version: + type: string + registration_token: + description: |- + RegistrationToken gates *creating* a new Fin identity. It is a + coarse, admin/instance-level shared secret (configured server-side as + FIN_REGISTRATION_TOKEN, distributed out-of-band), sent as a plain + body field because registration is the one call that happens before + a Fin has any credential of its own to put in a header. + type: string + required: + - capabilities + - registration_token + type: object + fin.RegisterResponse: + properties: + fin_id: + type: string + fin_token: + type: string + job_lease_seconds: + type: integer + long_poll_timeout_seconds: + type: integer + poll_interval_seconds: + description: |- + PollIntervalSeconds/LongPollTimeoutSeconds/JobLeaseSeconds are + server-chosen operational defaults, echoed back so a Fin doesn't need + its own hardcoded copy of them. + type: integer + type: object + fin.ResultRequest: + properties: + error: + type: string + state: + $ref: '#/definitions/fin.JobState' + target_results: + items: + $ref: '#/definitions/fin.TargetResult' + type: array + variables: + $ref: '#/definitions/cacao.Variables' + required: + - state + type: object + fin.StatusPingRequest: + properties: + progress: + type: string + type: object + fin.StatusPingResponse: + properties: + action: + type: string + type: object + fin.StepInfo: + properties: + delay: + type: integer + description: + type: string + name: + type: string + timeout: + type: integer + type: object + fin.TargetResult: + properties: + error: + type: string + failed_command_index: + description: |- + FailedCommandIndex identifies which command in Commands aborted this + target's sequence, if any. + type: integer + state: + $ref: '#/definitions/fin.JobState' + target_index: + description: |- + TargetIndex identifies which entry in the job's Targets this result + is for. Nil when the job had no targets at all. + type: integer + variables: + $ref: '#/definitions/cacao.Variables' + type: object manual.ManualResponseStatus: enum: - success @@ -732,88 +984,260 @@ definitions: x-enum-varnames: - ManualResponseSuccessStatus - ManualResponseFailureStatus + pkg_models_fin.Capability: + properties: + description: + description: |- + Description is free text for humans (dashboards, logs) — it plays no + role in routing. + type: string + step_examples: + description: |- + StepExamples are optional, illustrative CACAO action steps showing + how a playbook author would invoke this capability (agent, commands, + targets, ...), surfaced to help playbook authors — they are never + interpreted or validated by SOARCA. A capability may offer more than + one example (e.g. to illustrate different commands or target shapes). + items: + $ref: '#/definitions/cacao.Step' + type: array + type: + description: |- + Type is the routing key: the value playbook authors write into + agent_definitions[...].type to target this capability pool. Multiple + independently-deployed Fin processes may register the same Type; + SOARCA treats them as one interchangeable pool and load-balances/ + fails over across them (see docs/adr/FIN-WEBHOOK-PROTOCOL-PROPOSAL.md + §2.1b). + type: string + version: + description: |- + Version is the capability implementation's own version string, for + operator/debugging visibility only. + type: string + required: + - type + type: object info: contact: {} title: SOARCA API version: 1.0.0 paths: - /keymanagement/: + /fin/: + delete: + description: delete this Fin's own registration. The fin_id is inferred from + the fin_token presented in the Authorization header - a Fin can only ever + delete its own registration, so it never needs to name itself explicitly. + produces: + - application/json + responses: + "204": + description: No Content + "404": + description: Not Found + schema: + $ref: '#/definitions/api.Error' + summary: delete this Fin's own registration + tags: + - fin get: - description: return all keys in the KMS + description: list all currently-registered fins and their capabilities produces: - application/json responses: "200": description: OK schema: - items: - type: string - type: array - "400": - description: Bad Request + $ref: '#/definitions/fin.ListResponse' + summary: list all currently-registered fins and their capabilities + tags: + - fin + /fin/{fin_id}: + delete: + description: forcibly remove a registered fin's record, e.g. one that is stale/offline + and will never come back to unregister itself. This is an admin/dashboard + action, not Fin-authenticated - unlike Unregister, it is not restricted to + a fin removing its own registration. + parameters: + - description: fin ID + in: path + name: fin_id + required: true + type: string + produces: + - application/json + responses: + "204": + description: No Content + "404": + description: Not Found schema: $ref: '#/definitions/api.Error' - summary: gets all keys from the KMS + summary: forcibly remove a registered fin (admin) tags: - - keymanagement - /keymanagement/:keyname/: - delete: - description: revokes the key by moving it to .revoked and renaming it + - fin + get: + description: look up a specific registered fin by id + parameters: + - description: fin ID + in: path + name: fin_id + required: true + type: string produces: - application/json responses: "200": description: OK schema: - type: json - "400": - description: Bad Request + $ref: '#/definitions/fin.Record' + "404": + description: Not Found schema: $ref: '#/definitions/api.Error' - summary: remove key from KMS + summary: look up a specific registered fin by id tags: - - keymanagement + - fin + /fin/jobs/{job_id}: put: - description: adds a key to the KMS; load key into cache and write file + consumes: + - application/json + description: submit the result of a claimed job. Only the Fin the job is currently + leased to may submit a result for it. parameters: - - description: key + - description: job ID + in: path + name: job_id + required: true + type: string + - description: job result in: body name: data required: true schema: - $ref: '#/definitions/api.KeyManagementKey' + $ref: '#/definitions/fin.ResultRequest' + produces: + - application/json + responses: + "204": + description: No Content + "400": + description: Bad Request + schema: + $ref: '#/definitions/api.Error' + "403": + description: Forbidden + schema: + $ref: '#/definitions/api.Error' + "404": + description: Not Found + schema: + $ref: '#/definitions/api.Error' + summary: submit the result of a claimed job + tags: + - fin + /fin/jobs/{job_id}/status: + patch: + consumes: + - application/json + description: extend a claimed job's lease (so a legitimately long-running job + isn't requeued out from under the Fin still working on it), and check for + a pending instruction such as cancellation + parameters: + - description: job ID + in: path + name: job_id + required: true + type: string + - description: progress + in: body + name: data + schema: + $ref: '#/definitions/fin.StatusPingRequest' produces: - application/json responses: "200": description: OK schema: - type: json + $ref: '#/definitions/fin.StatusPingResponse' "400": description: Bad Request schema: $ref: '#/definitions/api.Error' - summary: add key to KMS + "403": + description: Forbidden + schema: + $ref: '#/definitions/api.Error' + "404": + description: Not Found + schema: + $ref: '#/definitions/api.Error' + summary: extend a claimed job's lease, and check for a pending cancellation + instruction tags: - - keymanagement - /keymanagement/refresh/: + - fin + /fin/poll: post: - description: refresh the KMS and re-parse the underlying directory + consumes: + - application/json + description: long-poll for the next job matching this Fin's registered capability + types. Returns 204 No Content if long_poll_timeout_seconds elapses with no + job available - callers should simply poll again. + parameters: + - description: poll hints + in: body + name: data + schema: + $ref: '#/definitions/fin.PollRequest' produces: - application/json responses: "200": description: OK schema: - type: json + $ref: '#/definitions/fin.PollResponse' + "204": + description: No Content + "401": + description: Unauthorized + schema: + $ref: '#/definitions/api.Error' + summary: long-poll for the next job matching this Fin's registered capability + types + tags: + - fin + /fin/register: + post: + consumes: + - application/json + description: register a new Fin process, declaring the capability types it can + execute, and obtain its fin_id/fin_token + parameters: + - description: registration + in: body + name: data + required: true + schema: + $ref: '#/definitions/fin.RegisterRequest' + produces: + - application/json + responses: + "201": + description: Created + schema: + $ref: '#/definitions/fin.RegisterResponse' "400": description: Bad Request schema: $ref: '#/definitions/api.Error' - summary: refresh the KMS system, which allows the user to include manually added - files in the KMS + "403": + description: Forbidden + schema: + $ref: '#/definitions/api.Error' + summary: register a new Fin and obtain its fin_token tags: - - keymanagement + - fin /manual/: get: consumes: @@ -837,7 +1261,7 @@ paths: summary: get all pending manual commands that still needs values to be returned tags: - manual - /manual/{exec_id}/{step_id}: + /manual/{exec_id}/{step_execution_id}: get: consumes: - application/json @@ -848,9 +1272,11 @@ paths: name: exec_id required: true type: string - - description: step ID + - description: step execution ID (identifies a specific pending step invocation; + see GET /manual/ to discover it, as multiple pending commands may share + the same step ID) in: path - name: step_id + name: step_execution_id required: true type: string produces: @@ -867,23 +1293,27 @@ paths: summary: get a specific manual command that still needs a value to be returned tags: - manual - /manual/continue: - post: + put: consumes: - application/json - description: updates the value of a variable according to the manual interaction + description: |- + resolve a specific pending manual command by supplying its out args. This is a PUT + on the same resource GET /manual/{exec_id}/{step_execution_id} identifies, not a + generic RPC-style action, so the ids live in the path, not the body. parameters: - description: execution ID in: path name: exec_id required: true type: string - - description: step ID + - description: step execution ID (identifies a specific pending step invocation; + see GET /manual/ to discover it, as multiple pending commands may share + the same step ID) in: path - name: step_id + name: step_execution_id required: true type: string - - description: playbook + - description: resolution in: body name: data required: true @@ -900,7 +1330,7 @@ paths: description: Bad Request schema: $ref: '#/definitions/api.Error' - summary: updates the value of a variable according to the manual interaction + summary: resolve a specific pending manual command by supplying its out args tags: - manual /playbook/: diff --git a/examples/assignment-playbook.json b/examples/assignment-playbook.json index 4eca18da..4ca33b18 100644 --- a/examples/assignment-playbook.json +++ b/examples/assignment-playbook.json @@ -157,7 +157,7 @@ }, "agent_definitions": { "soarca--d6f577d5-9f6d-4318-8f12-bf5caaaa0ea9": { - "type": "soarca", + "type": "soarca-http-api", "name": "soarca-http-api" } }, diff --git a/examples/fin-playbook.json b/examples/fin-playbook.json index 3c5980a2..e89bf17e 100644 --- a/examples/fin-playbook.json +++ b/examples/fin-playbook.json @@ -1,42 +1,105 @@ { "type": "playbook", "spec_version": "cacao-2.0", - "id": "playbook--3c9885e6-95cc-49cb-9044-0e1d2c22be4a", - "name": "New Playbook1", - "created": "2025-06-23T09:41:13.868Z", - "modified": "2023-11-01T15:33:31.072Z", - "revoked": false, - "priority": 0, - "severity": 0, - "impact": 0, - "workflow_start": "start--b6c62023-dbd4-422e-adf9-505277f29a79", + "id": "playbook--7d9d5f52-6a91-4c8f-8b9c-9c1d4c3a8b6e", + "name": "Example Fin", + "description": "This playbook demonstrates combining a native SSH capability step with a step routed to an externally-registered Fin capability, via the HTTP/JSON Fin protocol (see docs/adr/FIN-WEBHOOK-PROTOCOL-PROPOSAL.md and https://soarca.io/docs/soarca-extensions/fin-protocol)", + "playbook_types": [ + "notification" + ], + "created_by": "identity--96abab60-238a-44ff-8962-5806aa60cbce", + "created": "2023-11-20T15:56:00.123456Z", + "modified": "2023-11-20T15:56:00.123456Z", + "valid_from": "2023-11-20T15:56:00.123456Z", + "valid_until": "2123-11-20T15:56:00.123456Z", + "priority": 1, + "severity": 1, + "impact": 1, + "labels": [ + "soarca", + "fin", + "example" + ], + "authentication_info_definitions": { + "user-auth--b7ddc2ea-9f6a-4e82-8eaa-be202e942090": { + "type": "user-auth", + "username": "root", + "password": "password" + } + }, "agent_definitions": { - "soarca-fin-pong-f896bb3b-bb37-429e-8ece-2d4286cf326d": { - "name": "soarca-fin-pong-f896bb3b-bb37-429e-8ece-2d4286cf326d", - "type": "net-address" + "soarca--00010001-1000-1000-a000-000100010001": { + "type": "soarca-ssh", + "name": "soarca-ssh" + }, + "custom-ssh-fin--f3f0194f-99e6-4966-8512-de3806fecfdf": { + "type": "custom-ssh-fin", + "name": "custom-ssh-fin--f3f0194f-99e6-4966-8512-de3806fecfdf" } }, - "created_by": "identity--112e9923-5c87-4cf2-9685-eda58a162a7a", - "description": "This is a template playbook containing only a start and end node", + "target_definitions": { + "ssh--1c3900b4-f86b-430d-b415-12312b9e31f4": { + "type": "ssh", + "name": "system 1", + "address": { + "ipv4": [ + "192.168.0.10" + ] + }, + "authentication_info": "user-auth--b7ddc2ea-9f6a-4e82-8eaa-be202e942090" + } + }, + "external_references": [ + { + "name": "TNO COSSAS", + "description": "TNO COSSAS", + "source": "TNO COSSAS", + "url": "https://cossas-project.org" + } + ], + "workflow_start": "start--9e7d62b2-88ac-4656-94e1-dbd4413ba008", + "workflow_exception": "end--a6f0b81e-affb-4bca-b4f6-a2d5af908958", "workflow": { - "start--b6c62023-dbd4-422e-adf9-505277f29a79": { + "start--9e7d62b2-88ac-4656-94e1-dbd4413ba008": { "type": "start", - "name": "Start", - "on_completion": "action--2fd6bca0-24ed-4d74-8eae-da126a84f7c4" + "name": "Start fin example", + "on_completion": "action--eb9372d4-d524-49fc-bf24-be26ea084779" + }, + "action--eb9372d4-d524-49fc-bf24-be26ea084779": { + "type": "action", + "name": "List directory over native SSH", + "description": "Executed by SOARCA's built-in SSH capability", + "on_completion": "action--88f4c4df-fa96-44e6-b310-1c06d193ea55", + "commands": [ + { + "type": "ssh", + "command": "ls -la" + } + ], + "targets": [ + "ssh--1c3900b4-f86b-430d-b415-12312b9e31f4" + ], + "agent": "soarca--00010001-1000-1000-a000-000100010001" }, - "action--2fd6bca0-24ed-4d74-8eae-da126a84f7c4": { + "action--88f4c4df-fa96-44e6-b310-1c06d193ea55": { "type": "action", - "agent": "soarca-fin-pong-f896bb3b-bb37-429e-8ece-2d4286cf326d", + "name": "Restart the nginx service via a Fin", + "description": "Routed to whichever registered Fin declares the custom-ssh-fin capability type - SOARCA does not run this step itself, it hands it off over the Fin protocol (register/poll/result) to an externally-deployed process", + "on_completion": "end--a6f0b81e-affb-4bca-b4f6-a2d5af908958", "commands": [ { - "type": "pong" + "type": "manual", + "command": "sudo systemctl restart nginx" } ], - "on_completion": "end--156d0625-26cb-458d-997b-c1f8a552cb3c" + "targets": [ + "ssh--1c3900b4-f86b-430d-b415-12312b9e31f4" + ], + "agent": "custom-ssh-fin--f3f0194f-99e6-4966-8512-de3806fecfdf" }, - "end--156d0625-26cb-458d-997b-c1f8a552cb3c": { + "end--a6f0b81e-affb-4bca-b4f6-a2d5af908958": { "type": "end", - "name": "End step" + "name": "End Flow" } } -} \ No newline at end of file +} diff --git a/examples/http-playbook.json b/examples/http-playbook.json index 4225506a..8becd1d9 100644 --- a/examples/http-playbook.json +++ b/examples/http-playbook.json @@ -22,7 +22,7 @@ ], "agent_definitions": { "soarca--00020001-1000-1000-a000-000100010001": { - "type": "soarca", + "type": "soarca-http-api", "name": "soarca-http-api" } }, diff --git a/examples/manual-playbook.json b/examples/manual-playbook.json index 95e8261f..db630a9b 100644 --- a/examples/manual-playbook.json +++ b/examples/manual-playbook.json @@ -68,10 +68,6 @@ } }, "agent_definitions": { - "soarca--00040001-1000-1000-a000-000100010001": { - "type": "soarca", - "name": "soarca-manual-capability" - }, "soarca-manual-capability--7b0e98db-fa93-42aa-8511-e871c65131b1": { "type": "soarca-manual", "name": "soarca-manual", diff --git a/examples/openc2-playbook.json b/examples/openc2-playbook.json index 256a07ec..cd6f580f 100644 --- a/examples/openc2-playbook.json +++ b/examples/openc2-playbook.json @@ -22,7 +22,7 @@ ], "agent_definitions": { "soarca--00020001-1000-1000-a000-000100010001": { - "type": "soarca", + "type": "soarca-openc2-http", "name": "soarca-openc2-http" } }, diff --git a/examples/powershell-playbook.json b/examples/powershell-playbook.json index 05fd5fe3..cc4f1adc 100644 --- a/examples/powershell-playbook.json +++ b/examples/powershell-playbook.json @@ -43,7 +43,7 @@ }, "agent_definitions": { "soarca--00040001-1000-1000-a000-000100010001": { - "type": "soarca", + "type": "soarca-powershell", "name": "soarca-powershell" } }, diff --git a/examples/security/ip-lookup.json b/examples/security/ip-lookup.json index 6be08177..aceb483c 100644 --- a/examples/security/ip-lookup.json +++ b/examples/security/ip-lookup.json @@ -346,7 +346,7 @@ }, "agent_definitions": { "soarca--29306bbd-ad5d-47c7-b62e-0e8ac44a6534": { - "type": "soarca", + "type": "soarca-http-api", "name": "soarca-http-api" } }, diff --git a/examples/ssh-playbook.json b/examples/ssh-playbook.json index c5689ba0..4aeab06f 100644 --- a/examples/ssh-playbook.json +++ b/examples/ssh-playbook.json @@ -29,7 +29,7 @@ }, "agent_definitions": { "soarca--00010001-1000-1000-a000-000100010001": { - "type": "soarca", + "type": "soarca-ssh", "name": "soarca-ssh" } }, diff --git a/go.mod b/go.mod index 2e86916f..99bced9f 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,9 @@ module soarca -go 1.25.0 +go 1.25.7 require ( github.com/COSSAS/gauth v1.0.0 - github.com/eclipse/paho.mqtt.golang v1.4.3 github.com/gin-contrib/cors v1.7.1 github.com/gin-gonic/gin v1.10.0 github.com/go-playground/assert/v2 v2.2.0 @@ -12,16 +11,20 @@ require ( github.com/gofrs/uuid v4.4.0+incompatible github.com/google/uuid v1.6.0 github.com/itchyny/gojq v0.12.19 + github.com/jackc/pgx/v5 v5.10.0 github.com/joho/godotenv v1.5.1 github.com/masterzen/winrm v0.0.0-20240702205601-3fad6e106085 + github.com/pressly/goose/v3 v3.27.3 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 - github.com/sirupsen/logrus v1.9.3 - github.com/stretchr/testify v1.10.0 + github.com/sirupsen/logrus v1.9.4 + github.com/spf13/viper v1.21.0 + github.com/stretchr/testify v1.11.1 github.com/swaggo/files v1.0.1 github.com/swaggo/gin-swagger v1.6.0 - go.mongodb.org/mongo-driver v1.12.1 - golang.org/x/crypto v0.52.0 + github.com/swaggo/swag v1.16.1 + golang.org/x/crypto v0.54.0 + modernc.org/sqlite v1.57.0 ) require ( @@ -38,24 +41,28 @@ require ( github.com/cloudwego/iasm v0.2.0 // indirect github.com/coreos/go-oidc/v3 v3.11.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-jose/go-jose/v4 v4.0.5 // indirect - github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/logr v1.4.4 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.19.6 // indirect github.com/go-openapi/spec v0.20.4 // indirect github.com/go-openapi/swag v0.19.15 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/goccy/go-json v0.10.2 // indirect - github.com/golang/snappy v0.0.1 // indirect github.com/gorilla/securecookie v1.1.2 // indirect github.com/gorilla/sessions v1.4.0 // indirect - github.com/gorilla/websocket v1.5.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/itchyny/timefmt-go v0.1.8 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect github.com/jcmturner/gofork v1.7.6 // indirect @@ -64,34 +71,43 @@ require ( github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.13.6 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mailru/easyjson v0.7.6 // indirect github.com/masterzen/simplexml v0.0.0-20190410153822-31eea3082786 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/mfridman/interpolate v0.0.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect - github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/stretchr/objx v0.5.2 // indirect - github.com/swaggo/swag v1.16.1 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/sagikazarmark/locafero v0.11.0 // indirect + github.com/sethvargo/go-retry v0.4.0 // indirect + github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect + github.com/stretchr/objx v0.5.3 // indirect + github.com/subosito/gotenv v1.6.0 // indirect github.com/tidwall/transform v0.0.0-20201103190739-32f242e2dbde // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect - github.com/xdg-go/pbkdf2 v1.0.0 // indirect - github.com/xdg-go/scram v1.1.2 // indirect - github.com/xdg-go/stringprep v1.0.4 // indirect - github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect + go.uber.org/multierr v1.11.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.8.0 // indirect - golang.org/x/net v0.54.0 // indirect + golang.org/x/net v0.57.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect - golang.org/x/sync v0.20.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/text v0.37.0 // indirect - golang.org/x/tools v0.44.0 // indirect - google.golang.org/protobuf v1.34.1 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect + golang.org/x/tools v0.47.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + modernc.org/libc v1.74.4 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect ) diff --git a/go.sum b/go.sum index 7c73f52e..c4930760 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/eclipse/paho.mqtt.golang v1.4.3 h1:2kwcUGn8seMUfWndX0hGbvH8r7crgcJguQNCyp70xik= -github.com/eclipse/paho.mqtt.golang v1.4.3/go.mod h1:CSYvoAlsMkhYOXh/oKyxa8EcBci6dVkLCbo5tTC1RIE= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/gin-contrib/cors v1.7.1 h1:s9SIppU/rk8enVvkzwiC2VK3UZ/0NNGsWfUKvV55rqs= @@ -44,8 +48,8 @@ github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA= -github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= -github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= @@ -64,18 +68,19 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= @@ -84,17 +89,25 @@ github.com/gorilla/securecookie v1.1.2/go.mod h1:NfCASbcHqRSY+3a8tlWJwsQap2VX5pw github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/sessions v1.4.0 h1:kpIYOp/oi6MG/p5PgxApU8srsSw9tuFbt46Lt7auzqQ= github.com/gorilla/sessions v1.4.0/go.mod h1:FLWm50oby91+hl7p/wRxDth9bWSuk0qVL2emc7lT5ik= -github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= -github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/itchyny/gojq v0.12.19 h1:ttXA0XCLEMoaLOz5lSeFOZ6u6Q3QxmG46vfgI4O0DEs= github.com/itchyny/gojq v0.12.19/go.mod h1:5galtVPDywX8SPSOrqjGxkBeDhSxEW1gSxoy7tn1iZY= github.com/itchyny/timefmt-go v0.1.8 h1:1YEo1JvfXeAHKdjelbYr/uCuhkybaHCeTkH8Bo791OI= github.com/itchyny/timefmt-go v0.1.8/go.mod h1:5E46Q+zj7vbTgWY8o5YkMeYb4I6GeWLFnetPy5oBrAI= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.10.0 h1:VhSvgU2jSli8o3AqIEOTJr7rZwAEUVo4E4XhR94Zfr0= +github.com/jackc/pgx/v5 v5.10.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -113,15 +126,13 @@ github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8Hm github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -136,33 +147,53 @@ github.com/masterzen/simplexml v0.0.0-20190410153822-31eea3082786 h1:2ZKn+w/BJeL github.com/masterzen/simplexml v0.0.0-20190410153822-31eea3082786/go.mod h1:kCEbxUJlNDEBNbdQMkPSp6yaKcRXVI6f4ddk8Riv4bc= github.com/masterzen/winrm v0.0.0-20240702205601-3fad6e106085 h1:PiQLLKX4vMYlJImDzJYtQScF2BbQ0GAjPIHCDqzHHHs= github.com/masterzen/winrm v0.0.0-20240702205601-3fad6e106085/go.mod h1:JajVhkiG2bYSNYYPYuWG7WZHr42CTjMTcCjfInRNCqc= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= +github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= -github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= -github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/pressly/goose/v3 v3.27.3 h1:pIglVHjw99r4e/hDHHwbl9vfOsDMqUokfkXo6+n/RxA= +github.com/pressly/goose/v3 v3.27.3/go.mod h1:Dag+xpV6o20HR2LFY1j0q6MDwc3f7vPUFDA77R+0yGY= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= +github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/santhosh-tekuri/jsonschema/v6 v6.0.1 h1:PKK9DyHxif4LZo+uQSgXNqs0jj5+xZwwfKHgph2lxBw= github.com/santhosh-tekuri/jsonschema/v6 v6.0.1/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sethvargo/go-retry v0.4.0 h1:9qy1OoIAxBL+gBYnkTnTnWle5wlfsXQlwRzIbbpdqPw= +github.com/sethvargo/go-retry v0.4.0/go.mod h1:tvsjdKG6xfiCx4LSiUZ06kcv38xvdVQwv8R6/VnnVWg= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= +github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -170,10 +201,10 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE= github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg= github.com/swaggo/gin-swagger v1.6.0 h1:y8sxvQ3E20/RCyrXeFfg60r6H0Z+SwpTjMYsMm+zy8M= @@ -186,79 +217,66 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= -github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= -github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= -github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= -github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= -github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.mongodb.org/mongo-driver v1.12.1 h1:nLkghSU8fQNaK7oUmDhQFsnrtcoNy7Z6LVFKsEecqgE= -go.mongodb.org/mongo-driver v1.12.1/go.mod h1:/rGBTebI3XYboVmgz+Wv3Bcbl3aD0QF9zl6kDDw18rQ= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/arch v0.8.0 h1:3wRIsP3pM4yUptoR96otTUOXI367OS0+c9eeRi9doIc= golang.org/x/arch v0.8.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= -golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= -golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= -golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= -golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= -golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= -google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -271,5 +289,33 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.29.1 h1:MKgdCV3WykTSPqpVrnxdEDS0HEd2FHpKZDzxzU5LyeI= +modernc.org/cc/v4 v4.29.1/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= +modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= +modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.74.4 h1:fX1Omw4o2/1C2iRkkIsrQTasJQldLhRmuPreXLoWs9k= +modernc.org/libc v1.74.4/go.mod h1:eeQAS9W3sZeKYMFubydxJpII9ybHWshk+7or7bLG9co= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.57.0 h1:qNQP6xnx5M0ISNtlnxoOX0+cD5bJ0/gr9aMmndFczzg= +modernc.org/sqlite v1.57.0/go.mod h1:yCJ2cmAaIkHQ25oXWrF8H4O1lIfPYPR26yCEDj2P3pQ= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= diff --git a/pkg/integration/thehive/cases/helper.go b/internal/adapters/thehive/cases/helper.go similarity index 87% rename from pkg/integration/thehive/cases/helper.go rename to internal/adapters/thehive/cases/helper.go index 81162e5a..2e419024 100644 --- a/pkg/integration/thehive/cases/helper.go +++ b/internal/adapters/thehive/cases/helper.go @@ -1,8 +1,8 @@ package cases import ( - "soarca/pkg/integration/thehive/common/models" - "soarca/pkg/models/cacao" + "soarca/internal/adapters/thehive/common/models" + "soarca/pkg/cacao" ) func CreateHiveObservables(variables cacao.Variables) models.Observables { diff --git a/pkg/integration/thehive/cases/thehive_cases.go b/internal/adapters/thehive/cases/thehive_cases.go similarity index 60% rename from pkg/integration/thehive/cases/thehive_cases.go rename to internal/adapters/thehive/cases/thehive_cases.go index ef80fda9..cd08f2de 100644 --- a/pkg/integration/thehive/cases/thehive_cases.go +++ b/internal/adapters/thehive/cases/thehive_cases.go @@ -3,11 +3,11 @@ package cases import ( "reflect" "soarca/internal/logger" - "soarca/pkg/integration/thehive/common/connector" - thehive_models "soarca/pkg/integration/thehive/common/models" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" - "soarca/pkg/reporting/cases" + "soarca/internal/adapters/thehive/common/connector" + thehive_models "soarca/internal/adapters/thehive/common/models" + "soarca/pkg/cacao" + "soarca/internal/runs/model" + "soarca/internal/reporting/cases" "time" "github.com/google/uuid" @@ -30,7 +30,7 @@ func NewCaseManager(connector connector.ITheHiveConnector) *HiveCaseManager { return &HiveCaseManager{connector: connector} } -func (manager *HiveCaseManager) AddToExistingOrCreateNew(meta execution.Metadata, +func (manager *HiveCaseManager) AddToExistingOrCreateNew(meta run.Metadata, playbook cacao.Playbook) cacao.Variable { //convert variables to observables @@ -45,7 +45,7 @@ func (manager *HiveCaseManager) AddToExistingOrCreateNew(meta execution.Metadata } else { if len(cases) > 0 { caseId = cases[0].ID - exe := thehive_models.ExecutionMetadata{ExecutionId: meta.ExecutionId.String(), + exe := thehive_models.RunMetadata{RunId: meta.RunId.String(), Playbook: playbook} if err := manager.connector.SetMapping(exe, caseId); err != nil { log.Error(err) @@ -55,9 +55,9 @@ func (manager *HiveCaseManager) AddToExistingOrCreateNew(meta execution.Metadata } } if caseId == "" { - newCaseId, err := manager.connector.PostNewExecutionCase( - thehive_models.ExecutionMetadata{ - ExecutionId: meta.ExecutionId.String(), + newCaseId, err := manager.connector.PostNewRunCase( + thehive_models.RunMetadata{ + RunId: meta.RunId.String(), Playbook: playbook, }, time.Now(), @@ -91,11 +91,11 @@ func (manager *HiveCaseManager) ConnectorTest() string { } // Creates a new *case* in The Hive with related triggering metadata -func (manager *HiveCaseManager) ReportWorkflowStart(executionId uuid.UUID, playbook cacao.Playbook, at time.Time) error { +func (manager *HiveCaseManager) ReportWorkflowStart(runId uuid.UUID, playbook cacao.Playbook, at time.Time) error { log.Trace("TheHive casesreporting workflow start") - // _, err := manager.connector.PostNewExecutionCase( - // thehive_models.ExecutionMetadata{ - // ExecutionId: executionId.String(), + // _, err := manager.connector.PostNewRunCase( + // thehive_models.RunMetadata{ + // RunId: runId.String(), // Playbook: playbook, // }, // at, @@ -103,14 +103,14 @@ func (manager *HiveCaseManager) ReportWorkflowStart(executionId uuid.UUID, playb return nil } -// Marks case closure according to workflow execution. Also reports all variables, and data -func (manager *HiveCaseManager) ReportWorkflowEnd(executionId uuid.UUID, playbook cacao.Playbook, workflowErr error, at time.Time) error { +// Marks case closure according to workflow run. Also reports all variables, and data +func (manager *HiveCaseManager) ReportWorkflowEnd(runId uuid.UUID, playbook cacao.Playbook, workflowErr error, at time.Time) error { log.Trace("TheHive casesreporting workflow end") - _, err := manager.connector.UpdateEndExecutionCase( - thehive_models.ExecutionMetadata{ - ExecutionId: executionId.String(), + _, err := manager.connector.UpdateEndRunCase( + thehive_models.RunMetadata{ + RunId: runId.String(), Variables: playbook.PlaybookVariables, - ExecutionErr: workflowErr, + RunErr: workflowErr, }, at, ) @@ -118,11 +118,11 @@ func (manager *HiveCaseManager) ReportWorkflowEnd(executionId uuid.UUID, playboo } // Adds *event* to case -func (manager *HiveCaseManager) ReportStepStart(executionId uuid.UUID, step cacao.Step, stepResults cacao.Variables, at time.Time) error { +func (manager *HiveCaseManager) ReportStepStart(metadata run.Metadata, step cacao.Step, stepResults cacao.Variables, at time.Time) error { log.Trace("TheHive casesreporting step start") _, err := manager.connector.UpdateStartStepTaskInCase( - thehive_models.ExecutionMetadata{ - ExecutionId: executionId.String(), + thehive_models.RunMetadata{ + RunId: metadata.RunId.String(), Step: step, }, at, @@ -130,15 +130,15 @@ func (manager *HiveCaseManager) ReportStepStart(executionId uuid.UUID, step caca return err } -// Populates event with step execution information -func (manager *HiveCaseManager) ReportStepEnd(executionId uuid.UUID, step cacao.Step, stepResults cacao.Variables, stepErr error, at time.Time) error { +// Populates event with step run information +func (manager *HiveCaseManager) ReportStepEnd(metadata run.Metadata, step cacao.Step, stepResults cacao.Variables, stepErr error, at time.Time) error { log.Trace("TheHive casesreporting step end") _, err := manager.connector.UpdateEndStepTaskInCase( - thehive_models.ExecutionMetadata{ - ExecutionId: executionId.String(), + thehive_models.RunMetadata{ + RunId: metadata.RunId.String(), Step: step, Variables: stepResults, - ExecutionErr: stepErr, + RunErr: stepErr, }, at, ) diff --git a/pkg/integration/thehive/common/connector/connector.go b/internal/adapters/thehive/common/connector/client.go similarity index 73% rename from pkg/integration/thehive/common/connector/connector.go rename to internal/adapters/thehive/common/connector/client.go index 9456eb7a..96816771 100644 --- a/pkg/integration/thehive/common/connector/connector.go +++ b/internal/adapters/thehive/common/connector/client.go @@ -7,10 +7,10 @@ import ( "net/http" "reflect" "soarca/internal/logger" - "soarca/pkg/integration/thehive/common/mappings" - thehive_models "soarca/pkg/integration/thehive/common/models" - thehive_utils "soarca/pkg/integration/thehive/common/utils" - "soarca/pkg/models/cacao" + "soarca/internal/adapters/thehive/common/mappings" + thehive_models "soarca/internal/adapters/thehive/common/models" + thehive_utils "soarca/internal/adapters/thehive/common/utils" + "soarca/pkg/cacao" "strings" "time" ) @@ -39,10 +39,10 @@ func init() { type ITheHiveConnector interface { Hello() string - PostNewExecutionCase(executionMetadata thehive_models.ExecutionMetadata, at time.Time) (string, error) - UpdateEndExecutionCase(executionMetadata thehive_models.ExecutionMetadata, at time.Time) (string, error) - UpdateStartStepTaskInCase(executionMetadata thehive_models.ExecutionMetadata, at time.Time) (string, error) - UpdateEndStepTaskInCase(executionMetadata thehive_models.ExecutionMetadata, at time.Time) (string, error) + PostNewRunCase(runMetadata thehive_models.RunMetadata, at time.Time) (string, error) + UpdateEndRunCase(runMetadata thehive_models.RunMetadata, at time.Time) (string, error) + UpdateStartStepTaskInCase(runMetadata thehive_models.RunMetadata, at time.Time) (string, error) + UpdateEndStepTaskInCase(runMetadata thehive_models.RunMetadata, at time.Time) (string, error) // GetCaseById(caseId string) (thehive_models.CaseResponse, error) @@ -51,7 +51,7 @@ type ITheHiveConnector interface { GetAllCases() error CreateObservableInCase(caseId string, observable thehive_models.Observable) error CreateCase(thisCase thehive_models.Case) (string, error) - SetMapping(meta thehive_models.ExecutionMetadata, caseId string) error + SetMapping(meta thehive_models.RunMetadata, caseId string) error } // ############################### TheHiveConnector object @@ -73,7 +73,7 @@ func completePath(host string) string { func NewConnector(theHiveEndpoint string, theHiveApiKey string, allowInsecure bool) *TheHiveConnector { ids_map := &mappings.SOARCATheHiveMap{} - ids_map.ExecutionsCaseMaps = map[string]mappings.ExecutionCaseMap{} + ids_map.RunsCaseMaps = map[string]mappings.RunCaseMap{} return &TheHiveConnector{ client: thehive_utils.SetupClient(allowInsecure), baseUrl: completePath(theHiveEndpoint), @@ -255,9 +255,9 @@ func (theHiveConnector *TheHiveConnector) UpdateCaseTags(caseId string, tags []s /// -func (theHiveConnector *TheHiveConnector) postCommentInTaskLog(executionId string, step cacao.Step, note string) error { - log.Trace(fmt.Sprintf("posting comment in task log via execution ID: %s. step ID: %s", executionId, step.ID)) - taskId, err := theHiveConnector.ids_map.RetrieveTaskId(executionId, step.ID) +func (theHiveConnector *TheHiveConnector) postCommentInTaskLog(runId string, step cacao.Step, note string) error { + log.Trace(fmt.Sprintf("posting comment in task log via run ID: %s. step ID: %s", runId, step.ID)) + taskId, err := theHiveConnector.ids_map.RetrieveTaskId(runId, step.ID) if err != nil { return err } @@ -274,12 +274,12 @@ func (theHiveConnector *TheHiveConnector) postCommentInTaskLog(executionId strin if err != nil { return err } - log.Trace(fmt.Sprintf("task log created. execution ID %s, Task id %s, message Id: %s", executionId, taskId, messageId)) + log.Trace(fmt.Sprintf("task log created. run ID %s, Task id %s, message Id: %s", runId, taskId, messageId)) return nil } -func (theHiveConnector *TheHiveConnector) postStepDataAsCommentInTaskLog(executionId string, step cacao.Step, note string) error { +func (theHiveConnector *TheHiveConnector) postStepDataAsCommentInTaskLog(runId string, step cacao.Step, note string) error { message := note + "\n" stepData, err := thehive_utils.StructToMDJSON(step) @@ -289,7 +289,7 @@ func (theHiveConnector *TheHiveConnector) postStepDataAsCommentInTaskLog(executi message = message + stepData - err = theHiveConnector.postCommentInTaskLog(executionId, step, message) + err = theHiveConnector.postCommentInTaskLog(runId, step, message) if err != nil { return err } @@ -297,7 +297,7 @@ func (theHiveConnector *TheHiveConnector) postStepDataAsCommentInTaskLog(executi return nil } -func (theHiveConnector *TheHiveConnector) postStepVariablesAsCommentInTaskLog(executionId string, step cacao.Step, note string) error { +func (theHiveConnector *TheHiveConnector) postStepVariablesAsCommentInTaskLog(runId string, step cacao.Step, note string) error { variablesString := note + "\n" for _, variable := range step.StepVariables { variableJson, err := thehive_utils.StructToMDJSON(variable) @@ -307,7 +307,7 @@ func (theHiveConnector *TheHiveConnector) postStepVariablesAsCommentInTaskLog(ex variablesString = variablesString + variableJson } - err := theHiveConnector.postCommentInTaskLog(executionId, step, variablesString) + err := theHiveConnector.postCommentInTaskLog(runId, step, variablesString) if err != nil { return err } @@ -315,9 +315,9 @@ func (theHiveConnector *TheHiveConnector) postStepVariablesAsCommentInTaskLog(ex return nil } -func (theHiveConnector *TheHiveConnector) postCommentInCase(executionId string, note string) error { - log.Trace(fmt.Sprintf("posting comment in case via execution ID: %s.", executionId)) - caseId, err := theHiveConnector.ids_map.RetrieveCaseId(executionId) +func (theHiveConnector *TheHiveConnector) postCommentInCase(runId string, note string) error { + log.Trace(fmt.Sprintf("posting comment in case via run ID: %s.", runId)) + caseId, err := theHiveConnector.ids_map.RetrieveCaseId(runId) if err != nil { return err } @@ -335,11 +335,11 @@ func (theHiveConnector *TheHiveConnector) postCommentInCase(executionId string, if err != nil { return err } - log.Trace(fmt.Sprintf("Case comment created. execution ID %s, caseId %s, message Id: %s", executionId, caseId, messageId)) + log.Trace(fmt.Sprintf("Case comment created. run ID %s, caseId %s, message Id: %s", runId, caseId, messageId)) return nil } -func (theHiveConnector *TheHiveConnector) postVariablesAsCommentInCase(executionId string, variables cacao.Variables, note string) error { +func (theHiveConnector *TheHiveConnector) postVariablesAsCommentInCase(runId string, variables cacao.Variables, note string) error { variablesString := note + "\n" for _, variable := range variables { @@ -350,7 +350,7 @@ func (theHiveConnector *TheHiveConnector) postVariablesAsCommentInCase(execution variablesString = variablesString + variableJson } - err := theHiveConnector.postCommentInCase(executionId, variablesString) + err := theHiveConnector.postCommentInCase(runId, variablesString) if err != nil { return err } @@ -358,8 +358,8 @@ func (theHiveConnector *TheHiveConnector) postVariablesAsCommentInCase(execution return nil } -func (theHiveConnector *TheHiveConnector) postNewStepTaskInCase(executionId string, step cacao.Step) error { - caseId, err := theHiveConnector.ids_map.RetrieveCaseId(executionId) +func (theHiveConnector *TheHiveConnector) postNewStepTaskInCase(runId string, step cacao.Step) error { + caseId, err := theHiveConnector.ids_map.RetrieveCaseId(runId) if err != nil { return err } @@ -381,16 +381,16 @@ func (theHiveConnector *TheHiveConnector) postNewStepTaskInCase(executionId stri if err != nil { return err } - theHiveConnector.ids_map.RegisterStepTaskInCase(executionId, step.ID, task_id) + theHiveConnector.ids_map.RegisterStepTaskInCase(runId, step.ID, task_id) return nil } // ######################################## Connector interface -func (theHiveConnector *TheHiveConnector) SetMapping(meta thehive_models.ExecutionMetadata, caseId string) error { - if err := theHiveConnector.ids_map.RegisterExecutionInCase(meta.ExecutionId, caseId); err != nil { - log.Error("failed to log execution in case") +func (theHiveConnector *TheHiveConnector) SetMapping(meta thehive_models.RunMetadata, caseId string) error { + if err := theHiveConnector.ids_map.RegisterRunInCase(meta.RunId, caseId); err != nil { + log.Error("failed to log run in case") log.Error(err) return err } @@ -399,7 +399,7 @@ func (theHiveConnector *TheHiveConnector) SetMapping(meta thehive_models.Executi log.Error(err) } - tags := []string{meta.ExecutionId, meta.Playbook.ID} + tags := []string{meta.RunId, meta.Playbook.ID} _, err = theHiveConnector.UpdateCaseTags(caseId, tags) if err != nil { log.Error(err) @@ -408,13 +408,13 @@ func (theHiveConnector *TheHiveConnector) SetMapping(meta thehive_models.Executi return err } -func (theHiveConnector *TheHiveConnector) PostNewExecutionCase(execMetadata thehive_models.ExecutionMetadata, at time.Time) (string, error) { - log.Trace(fmt.Sprintf("posting new case to The Hive. execution ID %s, playbook %+v", execMetadata.ExecutionId, execMetadata.Playbook)) +func (theHiveConnector *TheHiveConnector) PostNewRunCase(execMetadata thehive_models.RunMetadata, at time.Time) (string, error) { + log.Trace(fmt.Sprintf("posting new case to The Hive. run ID %s, playbook %+v", execMetadata.RunId, execMetadata.Playbook)) url := theHiveConnector.baseUrl + "/case" method := "POST" - // Add execution ID and playbook ID to tags (first and second tags) - caseTags := []string{execMetadata.ExecutionId, execMetadata.Playbook.ID} + // Add run ID and playbook ID to tags (first and second tags) + caseTags := []string{execMetadata.RunId, execMetadata.Playbook.ID} caseTags = append(caseTags, execMetadata.Playbook.Labels...) data := thehive_models.Case{ @@ -436,9 +436,9 @@ func (theHiveConnector *TheHiveConnector) PostNewExecutionCase(execMetadata theh return "", err } - log.Trace("Executing register execution in case") + log.Trace("Executing register run in case") - err = theHiveConnector.ids_map.RegisterExecutionInCase(execMetadata.ExecutionId, caseId) + err = theHiveConnector.ids_map.RegisterRunInCase(execMetadata.RunId, caseId) if err != nil { return "", err } @@ -448,29 +448,29 @@ func (theHiveConnector *TheHiveConnector) PostNewExecutionCase(execMetadata theh return caseId, err } -func (theHiveConnector *TheHiveConnector) populateCase(execMetadata thehive_models.ExecutionMetadata, at time.Time) error { +func (theHiveConnector *TheHiveConnector) populateCase(execMetadata thehive_models.RunMetadata, at time.Time) error { // Pre-populate tasks according to playbook steps for _, step := range execMetadata.Playbook.Workflow { if step.Type == cacao.StepTypeStart || step.Type == cacao.StepTypeEnd { continue } - err := theHiveConnector.postNewStepTaskInCase(execMetadata.ExecutionId, step) + err := theHiveConnector.postNewStepTaskInCase(execMetadata.RunId, step) if err != nil { return err } } - executionStartMessage := fmt.Sprintf( - "START\nplaybook ID\n\t\t[ %s ]\nexecution ID\n\t\t[ %s ]\nstarted at\n\t\t[ %s ]", - execMetadata.Playbook.ID, execMetadata.ExecutionId, at.String()) - err := theHiveConnector.postCommentInCase(execMetadata.ExecutionId, executionStartMessage) + runStartMessage := fmt.Sprintf( + "START\nplaybook ID\n\t\t[ %s ]\nrun ID\n\t\t[ %s ]\nstarted at\n\t\t[ %s ]", + execMetadata.Playbook.ID, execMetadata.RunId, at.String()) + err := theHiveConnector.postCommentInCase(execMetadata.RunId, runStartMessage) if err != nil { log.Warningf("could not post message to case: %s", err) } err = theHiveConnector.postVariablesAsCommentInCase( - execMetadata.ExecutionId, execMetadata.Playbook.PlaybookVariables, - "variables at start of execution") + execMetadata.RunId, execMetadata.Playbook.PlaybookVariables, + "variables at start of run") if err != nil { log.Warningf("could not report variables in case comment: %s", err) } @@ -478,28 +478,28 @@ func (theHiveConnector *TheHiveConnector) populateCase(execMetadata thehive_mode return err } -func (theHiveConnector *TheHiveConnector) UpdateEndExecutionCase(execMetadata thehive_models.ExecutionMetadata, at time.Time) (string, error) { - caseId, err := theHiveConnector.ids_map.RetrieveCaseId(execMetadata.ExecutionId) +func (theHiveConnector *TheHiveConnector) UpdateEndRunCase(execMetadata thehive_models.RunMetadata, at time.Time) (string, error) { + caseId, err := theHiveConnector.ids_map.RetrieveCaseId(execMetadata.RunId) if err != nil { return "", err } - log.Trace(fmt.Sprintf("updating case status to The Hive. execution ID %s, The Hive case ID %s", execMetadata.ExecutionId, caseId)) + log.Trace(fmt.Sprintf("updating case status to The Hive. run ID %s, The Hive case ID %s", execMetadata.RunId, caseId)) url := theHiveConnector.baseUrl + "/case/" + caseId method := "PATCH" - err = theHiveConnector.postVariablesAsCommentInCase(execMetadata.ExecutionId, execMetadata.Variables, "variables at end of execution") + err = theHiveConnector.postVariablesAsCommentInCase(execMetadata.RunId, execMetadata.Variables, "variables at end of run") if err != nil { log.Warningf("could not add task log: %s", err) } caseStatus := thehive_models.TheHiveCaseStatusTruePositive - closureComment := fmt.Sprintf("END\nexecution ID\n\t\t[ %s ]\nended at\n\t\t[ %s ]", execMetadata.ExecutionId, at.String()) - if execMetadata.ExecutionErr != nil { + closureComment := fmt.Sprintf("END\nrun ID\n\t\t[ %s ]\nended at\n\t\t[ %s ]", execMetadata.RunId, at.String()) + if execMetadata.RunErr != nil { caseStatus = thehive_models.TheHiveCaseStatusIndeterminate - closureComment = closureComment + fmt.Sprintf("execution error: %s", execMetadata.ExecutionErr) + closureComment = closureComment + fmt.Sprintf("run error: %s", execMetadata.RunErr) } - err = theHiveConnector.postCommentInCase(execMetadata.ExecutionId, closureComment) + err = theHiveConnector.postCommentInCase(execMetadata.RunId, closureComment) if err != nil { log.Warningf("could not add task log: %s", err) } @@ -519,9 +519,9 @@ func (theHiveConnector *TheHiveConnector) UpdateEndExecutionCase(execMetadata th // TODO: revise this function through -func (theHiveConnector *TheHiveConnector) UpdateStartStepTaskInCase(execMetadata thehive_models.ExecutionMetadata, at time.Time) (string, error) { - log.Trace(fmt.Sprintf("updating task in thehive. case ID %s. task started.", execMetadata.ExecutionId)) - taskId, err := theHiveConnector.ids_map.RetrieveTaskId(execMetadata.ExecutionId, execMetadata.Step.ID) +func (theHiveConnector *TheHiveConnector) UpdateStartStepTaskInCase(execMetadata thehive_models.RunMetadata, at time.Time) (string, error) { + log.Trace(fmt.Sprintf("updating task in thehive. case ID %s. task started.", execMetadata.RunId)) + taskId, err := theHiveConnector.ids_map.RetrieveTaskId(execMetadata.RunId, execMetadata.Step.ID) if err != nil { return "", err } @@ -557,16 +557,16 @@ func (theHiveConnector *TheHiveConnector) UpdateStartStepTaskInCase(execMetadata return "", err } - executionStartMessage := fmt.Sprintf( - "START\nexecution ID\t\t[ %s ]\nstep ID\t\t[ %s ]\nstarted at\t\t[ %s ]", - execMetadata.ExecutionId, execMetadata.Step.ID, at.String()) + runStartMessage := fmt.Sprintf( + "START\nrun ID\t\t[ %s ]\nstep ID\t\t[ %s ]\nstarted at\t\t[ %s ]", + execMetadata.RunId, execMetadata.Step.ID, at.String()) - err = theHiveConnector.postCommentInTaskLog(execMetadata.ExecutionId, execMetadata.Step, executionStartMessage) + err = theHiveConnector.postCommentInTaskLog(execMetadata.RunId, execMetadata.Step, runStartMessage) if err != nil { log.Warningf("could post message to task: %s", err) } - err = theHiveConnector.postStepDataAsCommentInTaskLog(execMetadata.ExecutionId, execMetadata.Step, "step data") + err = theHiveConnector.postStepDataAsCommentInTaskLog(execMetadata.RunId, execMetadata.Step, "step data") if err != nil { log.Warningf("could not report step data in step task log: %s", err) } @@ -574,9 +574,9 @@ func (theHiveConnector *TheHiveConnector) UpdateStartStepTaskInCase(execMetadata return theHiveConnector.getIdFromRespBody(body) } -func (theHiveConnector *TheHiveConnector) UpdateEndStepTaskInCase(execMetadata thehive_models.ExecutionMetadata, at time.Time) (string, error) { - log.Trace(fmt.Sprintf("updating task in thehive. case ID %s. task ended.", execMetadata.ExecutionId)) - taskId, err := theHiveConnector.ids_map.RetrieveTaskId(execMetadata.ExecutionId, execMetadata.Step.ID) +func (theHiveConnector *TheHiveConnector) UpdateEndStepTaskInCase(execMetadata thehive_models.RunMetadata, at time.Time) (string, error) { + log.Trace(fmt.Sprintf("updating task in thehive. case ID %s. task ended.", execMetadata.RunId)) + taskId, err := theHiveConnector.ids_map.RetrieveTaskId(execMetadata.RunId, execMetadata.Step.ID) if err != nil { return "", err } @@ -584,21 +584,21 @@ func (theHiveConnector *TheHiveConnector) UpdateEndStepTaskInCase(execMetadata t url := theHiveConnector.baseUrl + "/task/" + taskId method := "PATCH" - err = theHiveConnector.postStepVariablesAsCommentInTaskLog(execMetadata.ExecutionId, execMetadata.Step, "returned variables") + err = theHiveConnector.postStepVariablesAsCommentInTaskLog(execMetadata.RunId, execMetadata.Step, "returned variables") if err != nil { log.Warningf("could not report variables in step task log: %s", err) } taskStatus := thehive_models.TheHiveStatusCompleted - executionEndMessage := fmt.Sprintf( - "END\nexecution ID\t\t[ %s ]\nstep ID\t\t[ %s ]\nended at\t\t[ %s ]", - execMetadata.ExecutionId, execMetadata.Step.ID, at.String()) + runEndMessage := fmt.Sprintf( + "END\nrun ID\t\t[ %s ]\nstep ID\t\t[ %s ]\nended at\t\t[ %s ]", + execMetadata.RunId, execMetadata.Step.ID, at.String()) - if execMetadata.ExecutionErr != nil { + if execMetadata.RunErr != nil { taskStatus = thehive_models.TheHiveStatusCancelled - executionEndMessage = executionEndMessage + fmt.Sprintf("\nexecution error: %s", execMetadata.ExecutionErr) + runEndMessage = runEndMessage + fmt.Sprintf("\nrun error: %s", execMetadata.RunErr) } - err = theHiveConnector.postCommentInTaskLog(execMetadata.ExecutionId, execMetadata.Step, executionEndMessage) + err = theHiveConnector.postCommentInTaskLog(execMetadata.RunId, execMetadata.Step, runEndMessage) if err != nil { log.Warningf("could post message to task: %s", err) } diff --git a/internal/adapters/thehive/common/mappings/cases.go b/internal/adapters/thehive/common/mappings/cases.go new file mode 100644 index 00000000..53aad51c --- /dev/null +++ b/internal/adapters/thehive/common/mappings/cases.go @@ -0,0 +1,74 @@ +package mappings + +import ( + "fmt" + "reflect" + "soarca/internal/logger" +) + +var ( + component = reflect.TypeOf(RunCaseMap{}).PkgPath() + log *logger.Log +) + +func init() { + log = logger.Logger(component, logger.Info, "", logger.Json) +} + +// ############################### Playbook to TheHive ID mappings + +type SOARCATheHiveMap struct { + RunsCaseMaps map[string]RunCaseMap +} +type RunCaseMap struct { + caseId string + stepsTasksMap map[string]string +} + +// TODO: Change to using observables instead of updating the tasks descriptions + +func (soarcaTheHiveMap *SOARCATheHiveMap) CheckRunCaseExists(runId string) error { + if _, ok := soarcaTheHiveMap.RunsCaseMaps[runId]; !ok { + return fmt.Errorf("case not found for run id %s", runId) + } + return nil +} +func (soarcaTheHiveMap *SOARCATheHiveMap) CheckRunStepTaskExists(runId string, stepId string) error { + if _, ok := soarcaTheHiveMap.RunsCaseMaps[runId].stepsTasksMap[stepId]; !ok { + return fmt.Errorf("task not found for run id %s for step id %s", runId, stepId) + } + return nil +} + +func (soarcaTheHiveMap *SOARCATheHiveMap) RegisterRunInCase(runId string, caseId string) error { + soarcaTheHiveMap.RunsCaseMaps[runId] = RunCaseMap{ + caseId: caseId, + stepsTasksMap: map[string]string{}, + } + log.Info(fmt.Sprintf("registering run: %s, case id: %s", runId, caseId)) + + return nil +} +func (soarcaTheHiveMap *SOARCATheHiveMap) RegisterStepTaskInCase(runId string, stepId string, taskId string) { + soarcaTheHiveMap.RunsCaseMaps[runId].stepsTasksMap[stepId] = taskId +} + +func (soarcaTheHiveMap *SOARCATheHiveMap) RetrieveCaseId(runId string) (string, error) { + err := soarcaTheHiveMap.CheckRunCaseExists(runId) + if err != nil { + return "", err + } + return soarcaTheHiveMap.RunsCaseMaps[runId].caseId, nil +} + +func (soarcaTheHiveMap *SOARCATheHiveMap) RetrieveTaskId(runId string, stepId string) (string, error) { + err := soarcaTheHiveMap.CheckRunCaseExists(runId) + if err != nil { + return "", err + } + err = soarcaTheHiveMap.CheckRunStepTaskExists(runId, stepId) + if err != nil { + return "", err + } + return soarcaTheHiveMap.RunsCaseMaps[runId].stepsTasksMap[stepId], nil +} diff --git a/pkg/integration/thehive/common/models/response.go b/internal/adapters/thehive/common/models/response.go similarity index 100% rename from pkg/integration/thehive/common/models/response.go rename to internal/adapters/thehive/common/models/response.go diff --git a/pkg/integration/thehive/common/models/models.go b/internal/adapters/thehive/common/models/types.go similarity index 98% rename from pkg/integration/thehive/common/models/models.go rename to internal/adapters/thehive/common/models/types.go index 13f48ad0..7e09f92b 100644 --- a/pkg/integration/thehive/common/models/models.go +++ b/internal/adapters/thehive/common/models/types.go @@ -1,6 +1,6 @@ package models -import "soarca/pkg/models/cacao" +import "soarca/pkg/cacao" const ( TheHiveStatusInProgress = "InProgress" @@ -125,12 +125,12 @@ type Case struct { ObservableRule string `bson:"observableRule,omitempty" json:"observableRule,omitempty" validate:"max=128" example:"Observable rule"` } -type ExecutionMetadata struct { - ExecutionId string +type RunMetadata struct { + RunId string Playbook cacao.Playbook Step cacao.Step Variables cacao.Variables - ExecutionErr error + RunErr error } // Observable The Hive diff --git a/pkg/integration/thehive/common/utils/utils.go b/internal/adapters/thehive/common/utils/client.go similarity index 100% rename from pkg/integration/thehive/common/utils/utils.go rename to internal/adapters/thehive/common/utils/client.go diff --git a/pkg/integration/thehive/common/utils/http.go b/internal/adapters/thehive/common/utils/http.go similarity index 100% rename from pkg/integration/thehive/common/utils/http.go rename to internal/adapters/thehive/common/utils/http.go diff --git a/pkg/integration/thehive/reporter/thehive_reporter.go b/internal/adapters/thehive/reporter/thehive_reporter.go similarity index 50% rename from pkg/integration/thehive/reporter/thehive_reporter.go rename to internal/adapters/thehive/reporter/thehive_reporter.go index 7988d8cd..a6d6daa3 100644 --- a/pkg/integration/thehive/reporter/thehive_reporter.go +++ b/internal/adapters/thehive/reporter/thehive_reporter.go @@ -3,9 +3,10 @@ package thehive import ( "reflect" "soarca/internal/logger" - "soarca/pkg/integration/thehive/common/connector" - thehive_models "soarca/pkg/integration/thehive/common/models" - "soarca/pkg/models/cacao" + "soarca/internal/adapters/thehive/common/connector" + thehive_models "soarca/internal/adapters/thehive/common/models" + "soarca/pkg/cacao" + "soarca/internal/runs/model" "time" "github.com/google/uuid" @@ -33,11 +34,11 @@ func (theHiveReporter *TheHiveReporter) ConnectorTest() string { } // Creates a new *case* in The Hive with related triggering metadata -func (theHiveReporter *TheHiveReporter) ReportWorkflowStart(executionId uuid.UUID, playbook cacao.Playbook, at time.Time) error { +func (theHiveReporter *TheHiveReporter) ReportWorkflowStart(runId uuid.UUID, playbook cacao.Playbook, at time.Time) error { log.Trace("TheHive reporter reporting workflow start") - _, err := theHiveReporter.connector.PostNewExecutionCase( - thehive_models.ExecutionMetadata{ - ExecutionId: executionId.String(), + _, err := theHiveReporter.connector.PostNewRunCase( + thehive_models.RunMetadata{ + RunId: runId.String(), Playbook: playbook, }, at, @@ -45,14 +46,14 @@ func (theHiveReporter *TheHiveReporter) ReportWorkflowStart(executionId uuid.UUI return err } -// Marks case closure according to workflow execution. Also reports all variables, and data -func (theHiveReporter *TheHiveReporter) ReportWorkflowEnd(executionId uuid.UUID, playbook cacao.Playbook, workflowErr error, at time.Time) error { +// Marks case closure according to workflow run. Also reports all variables, and data +func (theHiveReporter *TheHiveReporter) ReportWorkflowEnd(runId uuid.UUID, playbook cacao.Playbook, workflowErr error, at time.Time) error { log.Trace("TheHive reporter reporting workflow end") - _, err := theHiveReporter.connector.UpdateEndExecutionCase( - thehive_models.ExecutionMetadata{ - ExecutionId: executionId.String(), + _, err := theHiveReporter.connector.UpdateEndRunCase( + thehive_models.RunMetadata{ + RunId: runId.String(), Variables: playbook.PlaybookVariables, - ExecutionErr: workflowErr, + RunErr: workflowErr, }, at, ) @@ -60,11 +61,11 @@ func (theHiveReporter *TheHiveReporter) ReportWorkflowEnd(executionId uuid.UUID, } // Adds *event* to case -func (theHiveReporter *TheHiveReporter) ReportStepStart(executionId uuid.UUID, step cacao.Step, stepResults cacao.Variables, at time.Time) error { +func (theHiveReporter *TheHiveReporter) ReportStepStart(metadata run.Metadata, step cacao.Step, stepResults cacao.Variables, at time.Time) error { log.Trace("TheHive reporter reporting step start") _, err := theHiveReporter.connector.UpdateStartStepTaskInCase( - thehive_models.ExecutionMetadata{ - ExecutionId: executionId.String(), + thehive_models.RunMetadata{ + RunId: metadata.RunId.String(), Step: step, }, at, @@ -72,15 +73,15 @@ func (theHiveReporter *TheHiveReporter) ReportStepStart(executionId uuid.UUID, s return err } -// Populates event with step execution information -func (theHiveReporter *TheHiveReporter) ReportStepEnd(executionId uuid.UUID, step cacao.Step, stepResults cacao.Variables, stepErr error, at time.Time) error { +// Populates event with step run information +func (theHiveReporter *TheHiveReporter) ReportStepEnd(metadata run.Metadata, step cacao.Step, stepResults cacao.Variables, stepErr error, at time.Time) error { log.Trace("TheHive reporter reporting step end") _, err := theHiveReporter.connector.UpdateEndStepTaskInCase( - thehive_models.ExecutionMetadata{ - ExecutionId: executionId.String(), + thehive_models.RunMetadata{ + RunId: metadata.RunId.String(), Step: step, Variables: stepResults, - ExecutionErr: stepErr, + RunErr: stepErr, }, at, ) diff --git a/internal/app/app_test.go b/internal/app/app_test.go new file mode 100644 index 00000000..fbfe33de --- /dev/null +++ b/internal/app/app_test.go @@ -0,0 +1,119 @@ +package app + +import ( + "errors" + "testing" + + "soarca/internal/config" + orchestrator "soarca/internal/orchestrator" + httptransport "soarca/internal/transport/http" + + "github.com/gin-gonic/gin" +) + +type fakeTransport struct { + setupCalled bool + runCalled bool +} + +func (f *fakeTransport) SetupServer() (*gin.Engine, error) { + f.setupCalled = true + return gin.New(), nil +} + +func (f *fakeTransport) RunServer(*gin.Engine) error { + f.runCalled = true + return nil +} + +func TestRunWiresRuntimeAndTransport(t *testing.T) { + origLoadConfig := loadConfig + origNewRuntime := newRuntime + origNewTransport := newTransport + t.Cleanup(func() { + loadConfig = origLoadConfig + newRuntime = origNewRuntime + newTransport = origNewTransport + }) + + cfg := config.Config{ + Server: config.ServerConfig{Port: "8080"}, + Storage: config.StorageConfig{ + DatabaseURL: "sqlite://:memory:", + }, + Fin: config.FinConfig{StaleAfterMultiplier: 1, LongPollTimeoutSeconds: 1}, + RunState: config.RunStateConfig{MaxRuns: 5}, + HTTP: config.HTTPConfig{SkipCertValidation: true}, + Auth: config.AuthConfig{Enabled: false}, + TheHive: config.TheHiveConfig{ + Activate: false, + }, + CORS: config.CORSConfig{AllowedOrigins: "*"}, + } + + var gotRuntimeOpts orchestrator.Options + var gotTransportOpts httptransport.Options + fake := &fakeTransport{} + + loadConfig = func() (config.Config, error) { + return cfg, nil + } + newRuntime = func(opts orchestrator.Options) (*orchestrator.Runtime, error) { + gotRuntimeOpts = opts + return &orchestrator.Runtime{}, nil + } + newTransport = func(ops orchestrator.Operations, opts httptransport.Options) transport { + gotTransportOpts = opts + return fake + } + + if err := Run(); err != nil { + t.Fatalf("Run() returned error: %v", err) + } + + if gotRuntimeOpts.Storage != cfg.Storage { + t.Fatalf("runtime options storage mismatch: %#v", gotRuntimeOpts.Storage) + } + if gotRuntimeOpts.RunState != cfg.RunState { + t.Fatalf("runtime options runstate mismatch: %#v", gotRuntimeOpts.RunState) + } + if gotRuntimeOpts.HTTP != cfg.HTTP { + t.Fatalf("runtime options http mismatch: %#v", gotRuntimeOpts.HTTP) + } + if gotRuntimeOpts.TheHive != cfg.TheHive { + t.Fatalf("runtime options thehive mismatch: %#v", gotRuntimeOpts.TheHive) + } + if gotTransportOpts.Server != cfg.Server { + t.Fatalf("transport server mismatch: %#v", gotTransportOpts.Server) + } + if gotTransportOpts.Fin != cfg.Fin { + t.Fatalf("transport fin mismatch: %#v", gotTransportOpts.Fin) + } + if gotTransportOpts.Auth != cfg.Auth { + t.Fatalf("transport auth mismatch: %#v", gotTransportOpts.Auth) + } + if gotTransportOpts.CORS != cfg.CORS { + t.Fatalf("transport cors mismatch: %#v", gotTransportOpts.CORS) + } + if !fake.setupCalled { + t.Fatal("expected SetupServer to be called") + } + if !fake.runCalled { + t.Fatal("expected RunServer to be called") + } +} + +func TestRunReturnsConfigLoadError(t *testing.T) { + origLoadConfig := loadConfig + t.Cleanup(func() { + loadConfig = origLoadConfig + }) + + loadConfig = func() (config.Config, error) { + return config.Config{}, errors.New("boom") + } + + if err := Run(); err == nil { + t.Fatal("expected Run to return an error") + } +} diff --git a/internal/app/boundary_test.go b/internal/app/boundary_test.go new file mode 100644 index 00000000..91d9acbd --- /dev/null +++ b/internal/app/boundary_test.go @@ -0,0 +1,158 @@ +package app + +import ( + "testing" + + "soarca/internal/config" + orchestrator "soarca/internal/orchestrator" + httptransport "soarca/internal/transport/http" +) + +// TestRuntimeBoundary verifies the runtime exposes its use cases as behaviour +// only. Operations must carry no infrastructure: if a store, runstate, queue or +// walker factory ever appears here, transport can reach through it again. +func TestRuntimeBoundary(t *testing.T) { + app, err := orchestrator.New(mockRuntimeOptions()) + if err != nil { + t.Fatalf("Failed to create runtime: %v", err) + } + defer app.Close() + + ops := app.Operations() + + if ops.Playbooks == nil { + t.Error("Operations.Playbooks is nil") + } + if ops.Runs == nil { + t.Error("Operations.Runs is nil") + } + if ops.Fins == nil { + t.Error("Operations.Fins is nil") + } + if ops.Work == nil { + t.Error("Operations.Work is nil") + } + if ops.Manual == nil { + t.Error("Operations.Manual is nil") + } + + // The runtime must not hand out infrastructure. If any of these compile, + // the boundary has been violated: + // _ = runtime.GetPlaybookStore() + // _ = runtime.GetCache() + // _ = runtime.GetFinQueue() +} + +// TestTransportBoundary verifies the transport layer only does route +// registration and startup, driven purely by Operations. +func TestTransportBoundary(t *testing.T) { + app, err := orchestrator.New(mockRuntimeOptions()) + if err != nil { + t.Fatalf("Failed to create runtime: %v", err) + } + defer app.Close() + + server := httptransport.New(app.Operations(), mockTransportOptions()) + + engine, err := server.SetupServer() + if err != nil { + t.Fatalf("Failed to setup server: %v", err) + } + if engine == nil { + t.Error("SetupServer returned nil engine") + } + if got := len(engine.Routes()); got == 0 { + t.Error("SetupServer registered no routes") + } + + // The transport must not expose runtime internals. If any of these + // compile, the boundary has been violated: + // _ = server.GetPlaybookStore() + // _ = server.NewWalker() +} + +// TestConfigBoundary verifies config is split by ownership: the transport is +// handed only what HTTP actually needs, never the whole application config. +func TestConfigBoundary(t *testing.T) { + runtimeOpts := mockRuntimeOptions() + + if runtimeOpts.RunState.MaxRuns != 5 { + t.Error("Runtime should own RunState config") + } + if runtimeOpts.TheHive.Activate != false { + t.Error("Runtime should own TheHive config") + } + if runtimeOpts.HTTP.SkipCertValidation != false { + t.Error("Runtime should own outbound HTTP config") + } + + transportOpts := mockTransportOptions() + + if transportOpts.Server.Port != "8080" { + t.Error("Transport should have Server config with port") + } + if transportOpts.Fin.RegistrationToken != "test-token" { + t.Error("Transport should have Fin config") + } + if transportOpts.Auth.Enabled != false { + t.Error("Transport should have Auth config") + } + if transportOpts.CORS.AllowedOrigins != "*" { + t.Error("Transport should have CORS config") + } + + // Transport options must not carry orchestrator concerns. If any of these + // compile, the split has regressed: + // _ = transportOpts.Storage + // _ = transportOpts.TheHive +} + +// Helper functions for testing + +func mockStorageConfig() config.StorageConfig { + return config.StorageConfig{ + DatabaseURL: "sqlite://:memory:", + } +} + +func mockRunStateConfig() config.RunStateConfig { + return config.RunStateConfig{ + MaxRuns: 5, + } +} + +func mockFinConfig() config.FinConfig { + return config.FinConfig{ + RegistrationToken: "test-token", + PollIntervalSeconds: 5, + LongPollTimeoutSeconds: 30, + JobLeaseSeconds: 60, + StaleAfter: 300, + } +} + +func mockRuntimeOptions() orchestrator.Options { + return orchestrator.Options{ + Storage: mockStorageConfig(), + RunState: mockRunStateConfig(), + Fin: mockFinConfig(), + HTTP: config.HTTPConfig{ + SkipCertValidation: false, + }, + TheHive: config.TheHiveConfig{ + Activate: false, + }, + } +} + +func mockTransportOptions() httptransport.Options { + return httptransport.Options{ + Server: config.ServerConfig{ + Port: "8080", + EnableTLS: false, + }, + Fin: mockFinConfig(), + Auth: config.AuthConfig{Enabled: false}, + CORS: config.CORSConfig{AllowedOrigins: "*"}, + } +} diff --git a/internal/app/run.go b/internal/app/run.go new file mode 100644 index 00000000..5f46131f --- /dev/null +++ b/internal/app/run.go @@ -0,0 +1,75 @@ +package app + +import ( + "reflect" + + "soarca/internal/config" + "soarca/internal/logger" + orchestrator "soarca/internal/orchestrator" + httptransport "soarca/internal/transport/http" + + "github.com/gin-gonic/gin" +) + +var log *logger.Log + +type Empty struct{} + +type transport interface { + SetupServer() (*gin.Engine, error) + RunServer(*gin.Engine) error +} + +var loadConfig = config.Load +var newRuntime = orchestrator.New +var newTransport = func(ops orchestrator.Operations, opts httptransport.Options) transport { + return httptransport.New(ops, opts) +} + +func init() { + log = logger.Logger(reflect.TypeOf(Empty{}).PkgPath(), logger.Info, "", logger.Json) +} + +// Run loads configuration, wires the orchestrator, and starts the HTTP server. +func Run() error { + cfg, err := loadConfig() + if err != nil { + log.Error("Failed to load configuration:", err) + return err + } + + cfg.LogSettings() + + app, err := newRuntime(orchestrator.Options{ + Storage: cfg.Storage, + RunState: cfg.RunState, + Fin: cfg.Fin, + HTTP: cfg.HTTP, + TheHive: cfg.TheHive, + }) + if err != nil { + log.Error("Failed to initialize application:", err) + return err + } + defer app.Close() + + server := newTransport(app.Operations(), httptransport.Options{ + Server: cfg.Server, + Fin: cfg.Fin, + Auth: cfg.Auth, + CORS: cfg.CORS, + }) + engine, err := server.SetupServer() + if err != nil { + log.Error("Failed to setup server:", err) + return err + } + + if err := server.RunServer(engine); err != nil { + log.Error("Failed to run server:", err) + return err + } + + log.Info("exit") + return nil +} diff --git a/internal/config/load.go b/internal/config/load.go new file mode 100644 index 00000000..bbb98221 --- /dev/null +++ b/internal/config/load.go @@ -0,0 +1,189 @@ +package config + +import ( + "errors" + "reflect" + "time" + + "soarca/internal/logger" + + "github.com/spf13/viper" +) + +var log *logger.Log + +// DefaultDatabaseURL keeps a development database in the working directory so a +// fresh checkout runs without any configuration. +const DefaultDatabaseURL = "sqlite://soarca.db" + +func init() { + log = logger.Logger(reflect.TypeOf(Config{}).PkgPath(), logger.Info, "", logger.Json) +} + +type Config struct { + Server ServerConfig + Storage StorageConfig + Fin FinConfig + RunState RunStateConfig + HTTP HTTPConfig + Auth AuthConfig + TheHive TheHiveConfig + CORS CORSConfig +} + +type ServerConfig struct { + Port string + EnableTLS bool + CertFile string + CertKey string +} + +type StorageConfig struct { + // DatabaseURL selects the backend by scheme, e.g. sqlite://soarca.db or + // postgres://user:pass@host:5432/soarca. + DatabaseURL string +} + +type FinConfig struct { + PollIntervalSeconds int + LongPollTimeoutSeconds int + JobLeaseSeconds int + StaleAfterMultiplier int + StaleAfter time.Duration // Computed from LongPollTimeoutSeconds * StaleAfterMultiplier + RegistrationToken string +} + +type RunStateConfig struct { + MaxRuns int +} + +type HTTPConfig struct { + SkipCertValidation bool +} + +type AuthConfig struct { + Enabled bool +} + +type TheHiveConfig struct { + Activate bool + APIToken string + APIBaseURL string + AllowInsecure bool + EnableReporter bool + EnableCaseManager bool +} + +type CORSConfig struct { + AllowedOrigins string +} + +// Load reads configuration from environment variables using Viper. +func Load() (Config, error) { + v := viper.New() + + // Bind environment variables. Viper's default replacer is a no-op, which + // keeps env keys exact; passing nil here would panic in getEnv. + v.AutomaticEnv() + + // Set defaults + v.SetDefault("PORT", "8080") + v.SetDefault("ENABLE_TLS", false) + v.SetDefault("CERT_FILE", "./certs/server.crt") + v.SetDefault("CERT_KEY_FILE", "./certs/server.key") + v.SetDefault("DATABASE_URL", DefaultDatabaseURL) + v.SetDefault("FIN_POLL_INTERVAL_SECONDS", 5) + v.SetDefault("FIN_LONG_POLL_TIMEOUT_SECONDS", 25) + v.SetDefault("FIN_JOB_LEASE_SECONDS", 60) + v.SetDefault("FIN_STALE_AFTER_MULTIPLIER", 2) + v.SetDefault("FIN_REGISTRATION_TOKEN", "") + v.SetDefault("MAX_RUNS", 10) + v.SetDefault("HTTP_SKIP_CERT_VALIDATION", false) + v.SetDefault("AUTH_ENABLED", false) + v.SetDefault("THEHIVE_ACTIVATE", false) + v.SetDefault("THEHIVE_ALLOW_INSECURE", true) + v.SetDefault("THEHIVE_REPORTER", false) + v.SetDefault("SOARCA_ALLOWED_ORIGINS", "*") + + cfg := Config{ + Server: ServerConfig{ + Port: v.GetString("PORT"), + EnableTLS: v.GetBool("ENABLE_TLS"), + CertFile: v.GetString("CERT_FILE"), + CertKey: v.GetString("CERT_KEY_FILE"), + }, + Storage: StorageConfig{ + DatabaseURL: v.GetString("DATABASE_URL"), + }, + Fin: FinConfig{ + PollIntervalSeconds: v.GetInt("FIN_POLL_INTERVAL_SECONDS"), + LongPollTimeoutSeconds: v.GetInt("FIN_LONG_POLL_TIMEOUT_SECONDS"), + JobLeaseSeconds: v.GetInt("FIN_JOB_LEASE_SECONDS"), + StaleAfterMultiplier: v.GetInt("FIN_STALE_AFTER_MULTIPLIER"), + RegistrationToken: v.GetString("FIN_REGISTRATION_TOKEN"), + }, + RunState: RunStateConfig{ + MaxRuns: v.GetInt("MAX_RUNS"), + }, + HTTP: HTTPConfig{ + SkipCertValidation: v.GetBool("HTTP_SKIP_CERT_VALIDATION"), + }, + Auth: AuthConfig{ + Enabled: v.GetBool("AUTH_ENABLED"), + }, + TheHive: TheHiveConfig{ + Activate: v.GetBool("THEHIVE_ACTIVATE"), + APIToken: v.GetString("THEHIVE_API_TOKEN"), + APIBaseURL: v.GetString("THEHIVE_API_BASE_URL"), + AllowInsecure: v.GetBool("THEHIVE_ALLOW_INSECURE"), + EnableReporter: v.GetBool("THEHIVE_REPORTER"), + EnableCaseManager: v.GetBool("THEHIVE_REPORTER"), // Alias to reporter + }, + CORS: CORSConfig{ + AllowedOrigins: v.GetString("SOARCA_ALLOWED_ORIGINS"), + }, + } + cfg.Fin.StaleAfter = time.Duration(cfg.Fin.LongPollTimeoutSeconds*cfg.Fin.StaleAfterMultiplier) * time.Second + + // Validate required fields + if err := cfg.Validate(); err != nil { + return Config{}, err + } + + return cfg, nil +} + +// Validate checks required configuration values. +func (c Config) Validate() error { + if c.Storage.DatabaseURL == "" { + return errors.New("DATABASE_URL must be set") + } + if c.Fin.LongPollTimeoutSeconds <= 0 { + return errors.New("FIN_LONG_POLL_TIMEOUT_SECONDS must be greater than 0") + } + if c.Fin.StaleAfterMultiplier <= 0 { + return errors.New("FIN_STALE_AFTER_MULTIPLIER must be greater than 0") + } + if c.Fin.StaleAfter <= 0 { + return errors.New("FIN staleAfter must be greater than 0") + } + if c.TheHive.Activate && (c.TheHive.APIToken == "" || c.TheHive.APIBaseURL == "") { + return errors.New("THEHIVE_ACTIVATE is enabled but THEHIVE_API_TOKEN or THEHIVE_API_BASE_URL are not set") + } + return nil +} + +// LogSettings logs the current configuration (sanitizing secrets). +func (c Config) LogSettings() { + log.Info("Configuration loaded:") + log.Infof(" Server Port: %s", c.Server.Port) + log.Infof(" Server TLS: %v", c.Server.EnableTLS) + log.Infof(" Storage: %s", c.Storage.DatabaseURL) + log.Infof(" RunState Size: %d", c.RunState.MaxRuns) + log.Infof(" Auth Enabled: %v", c.Auth.Enabled) + log.Infof(" TheHive Activated: %v", c.TheHive.Activate) +} + +func storageType(cfg StorageConfig) string { + return cfg.DatabaseURL +} diff --git a/internal/controller/controller.go b/internal/controller/controller.go deleted file mode 100644 index 69ebe33e..00000000 --- a/internal/controller/controller.go +++ /dev/null @@ -1,355 +0,0 @@ -package controller - -import ( - "errors" - "fmt" - "os" - "reflect" - "soarca/internal/database/memory" - "soarca/internal/logger" - - "soarca/pkg/core/capability" - "soarca/pkg/core/capability/fin/protocol" - "soarca/pkg/core/capability/http" - "soarca/pkg/core/capability/manual" - "soarca/pkg/core/capability/manual/interaction" - "soarca/pkg/core/capability/openc2" - "soarca/pkg/core/capability/powershell" - "soarca/pkg/core/capability/ssh" - "soarca/pkg/core/decomposer" - "soarca/pkg/core/executors/action" - "soarca/pkg/core/executors/condition" - "soarca/pkg/core/executors/playbook_action" - "soarca/pkg/extensions/soarca/assignment" - "soarca/pkg/reporting/cases" - "soarca/pkg/reporting/reporter" - "soarca/pkg/utils" - "soarca/pkg/utils/guid" - "soarca/pkg/utils/stix/expression/comparison" - "strconv" - "strings" - - finExecutor "soarca/pkg/core/capability/fin" - finChannelController "soarca/pkg/core/capability/fin/controller" - - thehiveCases "soarca/pkg/integration/thehive/cases" - "soarca/pkg/integration/thehive/common/connector" - thehive "soarca/pkg/integration/thehive/reporter" - - cache "soarca/pkg/reporting/reporter/downstream_reporter/cache" - - httpUtil "soarca/pkg/utils/http" - - timeUtil "soarca/pkg/utils/time" - - downstreamReporter "soarca/pkg/reporting/reporter/downstream_reporter" - - "github.com/COSSAS/gauth" - "github.com/gin-gonic/gin" - - mongo "soarca/internal/database/mongodb" - playbookrepository "soarca/internal/database/playbook" - routes "soarca/pkg/api" -) - -var log *logger.Log - -type Empty struct{} - -func init() { - log = logger.Logger(reflect.TypeOf(Empty{}).PkgPath(), logger.Info, "", logger.Json) -} - -type Controller struct { - finController finChannelController.IFinController - playbookRepo playbookrepository.IPlaybookRepository -} - -var mainController = Controller{} - -var mainCache = cache.Cache{} - -const defaultCacheSize int = 10 - -// One manual interaction per SOARCA instance -var mainInteraction = interaction.New(registerManualIntegration()) - -func (controller *Controller) NewDecomposer() decomposer.IDecomposer { - ssh := new(ssh.SshCapability) - capabilities := map[string]capability.ICapability{ssh.GetType(): ssh} - - skip, _ := strconv.ParseBool(utils.GetEnv("HTTP_SKIP_CERT_VALIDATION", "false")) - - httpUtil := new(httpUtil.HttpRequest) - httpUtil.SkipCertificateValidation(skip) - http := http.New(httpUtil) - capabilities[http.GetType()] = http - - openc2 := openc2.New(httpUtil) - capabilities[openc2.GetType()] = openc2 - - poswershell := powershell.New() - capabilities[poswershell.GetType()] = poswershell - - man := manual.New(mainInteraction) - capabilities[man.GetType()] = &man - - enableFins, _ := strconv.ParseBool(utils.GetEnv("ENABLE_FINS", "false")) - - if enableFins { - broker, port := getMqttDetails() - - finCapabilities := controller.finController.GetRegisteredCapabilities() - for key := range finCapabilities { - prot := protocol.New(&guid.Guid{}, protocol.Topic(key), protocol.Broker(broker), port) - fin := finExecutor.New(&prot) - capabilities[key] = fin - } - } - - // NOTE: Enrolling mainCache by default as reporter - reporter := reporter.New([]downstreamReporter.IDownStreamReporter{}) - downstreamReporters := []downstreamReporter.IDownStreamReporter{&mainCache} - - // Reporter integrations - - thehive_reporter, theHiveCaseManager := initializeIntegrationTheHiveReporting() - if thehive_reporter != nil { - downstreamReporters = append(downstreamReporters, thehive_reporter) - } - - reporter.RegisterReporters(downstreamReporters) - - soarcaTime := new(timeUtil.Time) - assignmentExtension := assignment.New() - actionExecutor := action.New(capabilities, reporter, soarcaTime, assignmentExtension) - playbookActionExecutor := playbook_action.New(controller, controller, reporter, soarcaTime) - stixComparison := comparison.New() - conditionExecutor := condition.New(stixComparison, reporter, soarcaTime) - guid := new(guid.Guid) - decompose := decomposer.New(actionExecutor, - playbookActionExecutor, - conditionExecutor, - guid, - reporter, - soarcaTime) - if theHiveCaseManager != nil { - decompose.SetCaseManager(theHiveCaseManager) - } - return decompose -} - -func (controller *Controller) setupDatabase() error { - initMongoDatabase, _ := strconv.ParseBool(utils.GetEnv("DATABASE", "false")) - - if initMongoDatabase { - - mongo.LoadComponent() - - log.Info("SOARCA API Trying to start") - uri := os.Getenv("MONGODB_URI") - username := os.Getenv("DB_USERNAME") - password := os.Getenv("DB_PASSWORD") - - if uri == "" || username == "" || password == "" { - log.Error("you must set 'MONGODB_URI' or 'DB_USERNAME' or 'DB_PASSWORD' in the environment variable") - return errors.New("could not obtain required environment settings") - } - err := mongo.SetupMongodb(uri, username, password) - if err != nil { - return err - } - controller.playbookRepo = playbookrepository.SetupPlaybookRepository(mongo.GetCacaoRepo(), mongo.DefaultLimitOpts()) - } else { - // Use in memory database - controller.playbookRepo = memory.New() - } - - return nil -} - -func (controller *Controller) GetDatabaseInstance() playbookrepository.IPlaybookRepository { - return controller.playbookRepo -} - -func Initialize() error { - app := gin.New() - log.Info("Log level is info") - log.Debug("Log level is debug") - log.Trace("Log level is trace") - - enableFins, _ := strconv.ParseBool(utils.GetEnv("ENABLE_FINS", "false")) - if enableFins { - if err := mainController.setupAndRunMqtt(); err != nil { - log.Error(err) - } - } - - cacheSize, _ := strconv.Atoi(utils.GetEnv("MAX_EXECUTIONS", strconv.Itoa(defaultCacheSize))) - mainCache = *cache.New(&timeUtil.Time{}, cacheSize) - - err := initializeCore(app) - if err != nil { - log.Error("Failed to init core") - return err - } - - err = run(app) - if err != nil { - log.Error("failed to run gin") - } - log.Info("exit") - return err -} - -func validateCertificates(certFile string, keyFile string) error { - _, err := os.Stat(certFile) - if os.IsNotExist(err) { - return fmt.Errorf("certificate file not found: %s", certFile) - } - - _, err = os.Stat(keyFile) - if os.IsNotExist(err) { - return fmt.Errorf("key file not found: %s", keyFile) - } - return nil -} - -func run(app *gin.Engine) error { - port := utils.GetEnv("PORT", "8080") - port = ":" + port - enableTLS, _ := strconv.ParseBool(utils.GetEnv("ENABLE_TLS", "false")) - certFile := utils.GetEnv("CERT_FILE", "./certs/server.crt") - keyFile := utils.GetEnv("CERT_KEY_FILE", "./certs/server.key") - - if enableTLS { - err := validateCertificates(certFile, keyFile) - if err != nil { - return fmt.Errorf("TLS configuration error: %w", err) - } - log.Infof("Starting HTTPS server on port %s", port) - return app.RunTLS(port, certFile, keyFile) - - } - - log.Infof("Starting HTTP server on port %s", port) - return app.Run(port) -} - -func initializeCore(app *gin.Engine) error { - origins := strings.Split(strings.ReplaceAll(utils.GetEnv("SOARCA_ALLOWED_ORIGINS", "*"), " ", ""), ",") - routes.Cors(app, origins) - - err := intializeAuthenticationMiddleware(app) - if err != nil { - log.Error("Failed to setup Authentication middleware") - return err - } - err = mainController.setupDatabase() - if err != nil { - log.Error("Failed to setup database:", err) - return err - } - - err = routes.Api(app, &mainController, &mainController) - if err != nil { - log.Error(err) - return err - } - - err = routes.Database(app, &mainController) - if err != nil { - log.Error(err) - return err - } - - // NOTE: Assuming that the cache is the main information mediator for - // the reporter API - err = routes.Reporter(app, &mainCache) - if err != nil { - log.Error(err) - return err - } - - // Manual capability native routes - routes.Manual(app, mainInteraction) - - routes.Logging(app) - routes.Swagger(app) - - return err -} - -func (controller *Controller) setupAndRunMqtt() error { - broker, port := getMqttDetails() - mqttClient := finChannelController.NewClient(protocol.Broker(broker), port) - finChannelController := finChannelController.New(*mqttClient) - controller.finController = finChannelController - err := finChannelController.ConnectAndSubscribe() - if err != nil { - log.Error(err) - return err - } - go finChannelController.Run() - return nil -} - -func registerManualIntegration() []interaction.IInteractionIntegrationNotifier { - // Manual interaction integrations will be initialized here when implemented - // Here we should check ENV variables, see if a manual interaction integration is selected, - // And populate the returned array via generating an instance of the notifier associated with - // the integration - which should be found in the integration code. - return []interaction.IInteractionIntegrationNotifier{} -} - -func initializeIntegrationTheHiveReporting() (downstreamReporter.IDownStreamReporter, cases.ICasesManager) { - initTheHiveReporter, _ := strconv.ParseBool(utils.GetEnv("THEHIVE_ACTIVATE", "false")) - if !initTheHiveReporter { - return nil, nil - } - log.Info("initializing The Hive reporting integration") - - thehiveApiToken := utils.GetEnv("THEHIVE_API_TOKEN", "") - thehiveApiBaseUrl := utils.GetEnv("THEHIVE_API_BASE_URL", "") - if len(thehiveApiBaseUrl) < 1 || len(thehiveApiToken) < 1 { - log.Warning("could not initialize The Hive reporting integration. Check to have configured the env variables correctly.") - return nil, nil - } - - theHiveInsecureConnection, _ := strconv.ParseBool(utils.GetEnv("THEHIVE_ALLOW_INSECURE", "true")) - - log.Info(fmt.Sprintf("creating new The hive connector with API base url at : %s", thehiveApiBaseUrl)) - theHiveConnector := connector.NewConnector(thehiveApiBaseUrl, thehiveApiToken, theHiveInsecureConnection) - caseReporting, _ := strconv.ParseBool(utils.GetEnv("THEHIVE_REPORTER", "false")) - if caseReporting { - log.Info("enabling the hive reporter") - theHiveCases := thehiveCases.NewCaseManager(theHiveConnector) - return theHiveCases, theHiveCases - } - theHiveReporter := thehive.NewReporter(theHiveConnector) - return theHiveReporter, nil -} - -func intializeAuthenticationMiddleware(app *gin.Engine) error { - authEnabled, _ := strconv.ParseBool(utils.GetEnv("AUTH_ENABLED", "false")) - if authEnabled { - auth, err := gauth.New(gauth.DefaultConfig()) - if err != nil { - log.Error("Failed to initialize authenticator:", err) - return err - } - app.Use(auth.LoadAuthContext()) - app.Use(auth.Middleware([]string{"soarca_admin"})) - - } - return nil -} - -func getMqttDetails() (string, int) { - broker := utils.GetEnv("MQTT_BROKER", "localhost") - port, err := strconv.Atoi(utils.GetEnv("MQTT_PORT", "1883")) - if err != nil { - port = 1883 - } - return broker, port -} diff --git a/internal/controller/database/controller_database.go b/internal/controller/database/controller_database.go deleted file mode 100644 index afe04d38..00000000 --- a/internal/controller/database/controller_database.go +++ /dev/null @@ -1,9 +0,0 @@ -package database - -import ( - playbookrepository "soarca/internal/database/playbook" -) - -type IController interface { - GetDatabaseInstance() playbookrepository.IPlaybookRepository -} diff --git a/internal/controller/decomposer_controller/controller_decomposer.go b/internal/controller/decomposer_controller/controller_decomposer.go deleted file mode 100644 index eab7b702..00000000 --- a/internal/controller/decomposer_controller/controller_decomposer.go +++ /dev/null @@ -1,9 +0,0 @@ -package decomposer_controller - -import ( - "soarca/pkg/core/decomposer" -) - -type IController interface { - NewDecomposer() decomposer.IDecomposer -} diff --git a/internal/controller/informer/execution_informer.go b/internal/controller/informer/execution_informer.go deleted file mode 100644 index 22848b60..00000000 --- a/internal/controller/informer/execution_informer.go +++ /dev/null @@ -1,12 +0,0 @@ -package informer - -import ( - "soarca/pkg/models/cache" - - "github.com/google/uuid" -) - -type IExecutionInformer interface { - GetExecutions() ([]cache.ExecutionEntry, error) - GetExecutionReport(executionKey uuid.UUID) (cache.ExecutionEntry, error) -} diff --git a/internal/database/database.go b/internal/database/database.go deleted file mode 100644 index a1a69230..00000000 --- a/internal/database/database.go +++ /dev/null @@ -1,13 +0,0 @@ -package database - -type Database interface { - Read(string) (any, error) - Find(map[string]string, ...interface{}) ([]any, error) - Create(interface{}) error - Update(string, interface{}) error - Delete(string) error -} -type FindOptions interface { - GetIds() interface{} - GetProjectionByType(interface{}) interface{} -} diff --git a/internal/database/memory/memory.go b/internal/database/memory/memory.go deleted file mode 100644 index ef0bddc8..00000000 --- a/internal/database/memory/memory.go +++ /dev/null @@ -1,84 +0,0 @@ -package memory - -import ( - "errors" - "soarca/pkg/models/api" - "soarca/pkg/models/cacao" - "soarca/pkg/models/decoder" -) - -type InMemoryDatabase struct { - playbooks map[string]cacao.Playbook -} - -func New() *InMemoryDatabase { - return &InMemoryDatabase{playbooks: make(map[string]cacao.Playbook)} -} - -func (memory *InMemoryDatabase) GetPlaybooks() ([]cacao.Playbook, error) { - size := len(memory.playbooks) - playbookList := make([]cacao.Playbook, 0, size) - for _, playbook := range memory.playbooks { - playbookList = append(playbookList, playbook) - } - - return playbookList, nil -} - -func (memory *InMemoryDatabase) GetPlaybookMetas() ([]api.PlaybookMeta, error) { - size := len(memory.playbooks) - playbookList := make([]api.PlaybookMeta, 0, size) - for _, playbook := range memory.playbooks { - meta := api.PlaybookMeta{ID: playbook.ID, Name: playbook.Name, - Description: playbook.Description, - ValidFrom: playbook.ValidFrom, - ValidUntil: playbook.ValidUntil, - Labels: playbook.Labels} - playbookList = append(playbookList, meta) - } - - return playbookList, nil -} - -func (memory *InMemoryDatabase) Create(json *[]byte) (cacao.Playbook, error) { - - if json == nil { - return cacao.Playbook{}, errors.New("empty input") - } - result := decoder.DecodeValidate(*json) - if result == nil { - return cacao.Playbook{}, errors.New("failed to decode") - } - _, ok := memory.playbooks[result.ID] - if ok { - return cacao.Playbook{}, errors.New("playbook already exists") - } - memory.playbooks[result.ID] = *result - return memory.playbooks[result.ID], nil -} - -func (memory *InMemoryDatabase) Read(id string) (cacao.Playbook, error) { - playbook, ok := memory.playbooks[id] - if !ok { - return cacao.Playbook{}, errors.New("playbook is not in repository") - } - return playbook, nil -} - -func (memory *InMemoryDatabase) Update(id string, json *[]byte) (cacao.Playbook, error) { - playbook, err := memory.Read(id) - if err != nil { - return playbook, err - } - updatePlaybook := decoder.DecodeValidate(*json) - if updatePlaybook == nil { - return cacao.Playbook{}, errors.New("failed to decode") - } - memory.playbooks[id] = *updatePlaybook - return *updatePlaybook, nil -} - -func (memory *InMemoryDatabase) Delete(id string) error { - delete(memory.playbooks, id) - return nil -} diff --git a/internal/database/memory/memory_test.go b/internal/database/memory/memory_test.go deleted file mode 100644 index d206aae5..00000000 --- a/internal/database/memory/memory_test.go +++ /dev/null @@ -1,246 +0,0 @@ -package memory - -import ( - "encoding/json" - "errors" - "fmt" - "io" - "os" - "soarca/pkg/models/cacao" - "soarca/pkg/models/decoder" - "sort" - "testing" - - "github.com/go-playground/assert/v2" -) - -var PB_PATH string = "../../../test/playbooks/" - -func TestCreate(t *testing.T) { - jsonFile, err := os.Open(PB_PATH + "playbook.json") - if err != nil { - fmt.Println(err) - t.Fail() - } - - byteValue, _ := io.ReadAll(jsonFile) - err = jsonFile.Close() - if err != nil { - fmt.Println("Not valid JSON") - t.Fail() - return - } - - var workflow = decoder.DecodeValidate(byteValue) - mem := New() - playbook, err := mem.Create(&byteValue) - assert.Equal(t, err, nil) - assert.Equal(t, playbook, workflow) - -} - -func TestRead(t *testing.T) { - jsonFile, err := os.Open(PB_PATH + "playbook.json") - if err != nil { - fmt.Println(err) - t.Fail() - } - - byteValue, _ := io.ReadAll(jsonFile) - err = jsonFile.Close() - if err != nil { - fmt.Println("Not valid JSON") - t.Fail() - return - } - - var workflow = decoder.DecodeValidate(byteValue) - - mem := New() - empty, err := mem.Read(workflow.ID) - assert.Equal(t, err, errors.New("playbook is not in repository")) - assert.Equal(t, empty, cacao.Playbook{}) - - playbook, err := mem.Create(&byteValue) - assert.Equal(t, err, nil) - result, err := mem.Read(playbook.ID) - assert.Equal(t, err, nil) - assert.Equal(t, playbook, result) - -} - -func TestUpdate(t *testing.T) { - jsonFile, err := os.Open(PB_PATH + "playbook.json") - if err != nil { - fmt.Println(err) - t.Fail() - } - - byteValue, _ := io.ReadAll(jsonFile) - err = jsonFile.Close() - - if err != nil { - fmt.Println("Not valid JSON") - t.Fail() - return - } - - var workflow = decoder.DecodeValidate(byteValue) - - mem := New() - empty, err := mem.Update(workflow.ID, nil) - assert.Equal(t, err, errors.New("playbook is not in repository")) - assert.Equal(t, empty, cacao.Playbook{}) - - playbook, err := mem.Create(&byteValue) - assert.Equal(t, err, nil) - - emptyBytes, err := json.Marshal(new([]byte)) - assert.Equal(t, err, nil) - - parsingFailed, err := mem.Update(playbook.ID, &emptyBytes) - assert.Equal(t, err, errors.New("failed to decode")) - assert.Equal(t, parsingFailed, cacao.Playbook{}) - - workflow.Description = "new" - jsonBytes, err := json.Marshal(workflow) - assert.Equal(t, err, nil) - - result, err := mem.Update(playbook.ID, &jsonBytes) - assert.Equal(t, err, nil) - assert.Equal(t, workflow, result) - -} - -func TestDelete(t *testing.T) { - jsonFile, err := os.Open(PB_PATH + "playbook.json") - if err != nil { - fmt.Println(err) - t.Fail() - } - - byteValue, _ := io.ReadAll(jsonFile) - err = jsonFile.Close() - if err != nil { - fmt.Println("Not valid JSON") - t.Fail() - return - } - - var workflow = decoder.DecodeValidate(byteValue) - - mem := New() - err = mem.Delete(workflow.ID) - assert.Equal(t, err, nil) - - playbook, err := mem.Create(&byteValue) - assert.Equal(t, err, nil) - assert.Equal(t, playbook, workflow) - readbackPlaybook, err := mem.Read(workflow.ID) - assert.Equal(t, err, nil) - assert.Equal(t, readbackPlaybook, workflow) - - err = mem.Delete(workflow.ID) - assert.Equal(t, err, nil) - - readbackPlaybook2, err := mem.Read(workflow.ID) - assert.Equal(t, err, errors.New("playbook is not in repository")) - assert.Equal(t, readbackPlaybook2, cacao.Playbook{}) - -} - -func TestGetAllPlaybooks(t *testing.T) { - jsonFile, err := os.Open(PB_PATH + "playbook.json") - if err != nil { - fmt.Println(err) - t.Fail() - } - - byteValue, _ := io.ReadAll(jsonFile) - err = jsonFile.Close() - if err != nil { - fmt.Println("Not valid JSON") - t.Fail() - return - } - - var workflow = decoder.DecodeValidate(byteValue) - - mem := New() - - list := []string{ - "playbook--f47d4081-21ed-4f21-9d05-6b368d73da30", - "playbook--d41b1046-9334-400b-baa9-91b2ea431731", - "playbook--586fa554-448a-427e-a719-938f6e033e0d", - "playbook--3b1e8e64-bb5c-426f-8086-b5a6f9c565e2", - "playbook--08a82149-d48d-45dd-b8a2-025713d74742", - "playbook--dcd634a4-04b5-4fa5-842b-bb842a104fbf", - "playbook--e848e4d2-f529-46e7-8dc4-6ef7acaad902", - "playbook--e56eb89e-e8ab-41b8-ba94-f97014670bc7", - "playbook--c2ae6f09-53c5-4e61-b7fb-f5207b2604b3", - "playbook--8dbdf991-2ec2-45d6-952d-7ef1ac2a9254", - } - sort.Strings(list) - - for _, id := range list { - workflow.ID = id - jsonBytes, err := json.Marshal(workflow) - assert.Equal(t, err, nil) - _, err = mem.Create(&jsonBytes) - assert.Equal(t, err, nil) - // assert.Equal(t, playbook, workflow) - } - - playbooks, err := mem.GetPlaybooks() - assert.Equal(t, err, nil) - assert.Equal(t, len(playbooks), 10) - -} - -func TestGetAllPlaybookMetas(t *testing.T) { - jsonFile, err := os.Open(PB_PATH + "playbook.json") - if err != nil { - fmt.Println(err) - t.Fail() - } - - byteValue, _ := io.ReadAll(jsonFile) - err = jsonFile.Close() - if err != nil { - fmt.Println("Not valid JSON") - t.Fail() - return - } - - var workflow = decoder.DecodeValidate(byteValue) - - mem := New() - - list := []string{ - "playbook--f47d4081-21ed-4f21-9d05-6b368d73da30", - "playbook--d41b1046-9334-400b-baa9-91b2ea431731", - "playbook--586fa554-448a-427e-a719-938f6e033e0d", - "playbook--3b1e8e64-bb5c-426f-8086-b5a6f9c565e2", - "playbook--08a82149-d48d-45dd-b8a2-025713d74742", - "playbook--dcd634a4-04b5-4fa5-842b-bb842a104fbf", - "playbook--e848e4d2-f529-46e7-8dc4-6ef7acaad902", - "playbook--e56eb89e-e8ab-41b8-ba94-f97014670bc7", - "playbook--c2ae6f09-53c5-4e61-b7fb-f5207b2604b3", - "playbook--8dbdf991-2ec2-45d6-952d-7ef1ac2a9254", - } - sort.Strings(list) - - for _, id := range list { - workflow.ID = id - jsonBytes, err := json.Marshal(workflow) - assert.Equal(t, err, nil) - _, err = mem.Create(&jsonBytes) - assert.Equal(t, err, nil) - // assert.Equal(t, playbook, workflow) - } - - playbooks, err := mem.GetPlaybookMetas() - assert.Equal(t, err, nil) - assert.Equal(t, len(playbooks), 10) - -} diff --git a/internal/database/mongodb/init.go b/internal/database/mongodb/init.go deleted file mode 100644 index e2130fe2..00000000 --- a/internal/database/mongodb/init.go +++ /dev/null @@ -1,15 +0,0 @@ -package mongodb - -import ( - "reflect" - - "soarca/internal/logger" -) - -var log *logger.Log - -type Empty struct{} - -func LoadComponent() { - log = logger.Logger(reflect.TypeOf(Empty{}).PkgPath(), logger.Trace, "", logger.Json) -} diff --git a/internal/database/mongodb/mongo.go b/internal/database/mongodb/mongo.go deleted file mode 100644 index 15964997..00000000 --- a/internal/database/mongodb/mongo.go +++ /dev/null @@ -1,269 +0,0 @@ -package mongodb - -import ( - "context" - "errors" - "reflect" - "time" - - "soarca/internal/database/projections" - cacao "soarca/pkg/models/cacao" - - "go.mongodb.org/mongo-driver/bson" - mongo "go.mongodb.org/mongo-driver/mongo" - options "go.mongodb.org/mongo-driver/mongo/options" -) - -const writeErrorDuplicationCode = 11000 - -var ( - cacaoPlayBookRepo *mongoCollection[cacao.Playbook] - mongoclient *mongo.Client -) - -type dbtypes interface { - cacao.Playbook // | for other supported types -} - -type mongoCollection[T dbtypes] struct { - Collection *mongo.Collection - collectionname string -} - -type mongoFindOptions struct { - findOptions *options.FindOptions -} - -// type additionalFindOptions func(*mongoFindOptions) - -func DefaultLimitOpts() mongoFindOptions { - return mongoFindOptions{ - findOptions: options.Find().SetSkip(0).SetLimit(100), - } -} - -func (mongoOpts mongoFindOptions) GetIds() interface{} { - return func(lo *mongoFindOptions) { - lo.findOptions.SetProjection(projections.Id.GetProjection()) - } -} - -func (mongoOpts mongoFindOptions) GetProjectionByType(interface{}) interface{} { - return func(lo *mongoFindOptions) { - lo.findOptions.SetProjection(projections.Meta.GetProjection()) - } -} - -func GetCacaoRepo() *mongoCollection[cacao.Playbook] { - return cacaoPlayBookRepo -} - -// func GetMongoClient() *mongodbClient { -// return mongoclient -// } - -func SetupMongodb(uri string, username string, password string) error { - log.Trace("Calling SetupMongodb() to start setting up the mongodb database implementation") - err := InitMongoClient(uri, username, password) - if err != nil { - log.Error("failed to setup MongoClient, error msg: ", err.Error()) - return err - } - - if mongoclient == nil { - const error_msg = "mongoclient is not set properly" - log.Error(error_msg) - return errors.New(error_msg) - } - - cacaoPlayBookRepo, err = NewMongoCollection[cacao.Playbook](mongoclient, "soarca", "cacoa_playbook_collection") - return err -} - -// helper function to poperly obtain whether object is already in the database store -func isDuplicate(err error) bool { - var e mongo.WriteException - if errors.As(err, &e) { - for _, we := range e.WriteErrors { - if we.Code == writeErrorDuplicationCode { // duplication code - return true - } - } - } - return false -} - -func (mongocollection *mongoCollection[T]) Read(id string) (any, error) { - var collection T - context, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - err := mongocollection.Collection.FindOne(context, bson.M{"_id": id}).Decode(&collection) - if err != nil { - log.Debug("Error in FindOne operation: ", err) - return nil, err - } - return collection, err -} - -func (mongocollection *mongoCollection[T]) Find(query map[string]string, findOps ...interface{}) ([]interface{}, error) { - opts := DefaultLimitOpts() - for _, fn := range findOps { - optionFunction := fn.(func(*mongoFindOptions)) - optionFunction(&opts) - } - - bsonQuery := bson.D{} - - for key, value := range query { - bsonQuery = append(bsonQuery, bson.E{Key: key, Value: value}) - } - - collection := make([]T, 0) - - // needs to do some hacking to get the required generic output format as any does not work. - context, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - cursors, err := mongocollection.Collection.Find(context, bsonQuery, opts.findOptions) - if err != nil { - log.Debug("Can not find object for given findOptions: ", err) - return nil, err - } - - err = cursors.All(context, &collection) - - if err != nil { - log.Debug("Cursor for objects error: ", err) - return nil, err - } - interfaceCollection := make([]interface{}, len(collection)) - for i, v := range collection { - interfaceCollection[i] = v - } - - return interfaceCollection, nil -} - -func (mongocollection *mongoCollection[T]) Create(data interface{}) error { - var t T - log.Trace("Attempting to insert object in collection: ", mongocollection.collectionname) - log.Trace(data) - input_type := reflect.TypeOf(data) // unpacking because input type of pointer type - comparison_type := reflect.TypeOf(t) - - if input_type != comparison_type { // checks if the input type provided is accordance with collection type - log.Error("Failed as input object does not match required collection type") - return errors.New("input type for Create function is not in accordance with required type") - } - // ctx := context.Background() - context, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - _, err := mongocollection.Collection.InsertOne(context, data) - - if isDuplicate(err) { - log.Debug("Detected duplication in collection: ", mongocollection.collectionname, context) - return errors.New("duplicate") - } - - return err -} - -func (mongocollection *mongoCollection[T]) Update(id string, data interface{}) error { - var collectionFind T - context, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - err := mongocollection.Collection.FindOne(context, bson.M{"_id": id}).Decode(&collectionFind) - if err != nil { - log.Debug(err) - return err - } - filter := bson.M{"_id": id} - update := bson.M{"$set": data} - - _, err = mongocollection.Collection.UpdateOne(context, filter, update) - if err != nil { - log.Debug(err) - } - return err -} - -func (mongocollection *mongoCollection[T]) Delete(id string) error { - log.Trace("Trying to delete object type: ", mongocollection.collectionname, "with id: ", id) - context, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - _, err := mongocollection.Collection.DeleteOne(context, bson.M{"_id": id}) - if err != nil { - log.Debug("Error in deleting document: ", err.Error()) - } - return err -} - -func NewMongoCollection[T dbtypes](mongo *mongo.Client, dbName string, colName string) (*mongoCollection[T], error) { - log.Trace("Setting a new Mongo Collection name: ", colName) - - if mongo == nil { - log.Error("nil pointer for mongoClient") - return nil, errors.New("nil pointer for mongoClient") - } - - if colName == "" { - log.Error("column name for NewMongoCollection not valid, because empty") - return nil, errors.New("database name for NewMongoCollection not valid, because empty") - } - - if dbName == "" { - log.Error("database name for NewMongoCollection not valid, because empty") - return nil, errors.New("database name for NewMongoCollection not valid, because empty") - } - - collection := mongo.Database(dbName).Collection(colName) - return &mongoCollection[T]{Collection: collection, collectionname: colName}, nil -} - -func InitMongoClient(mongo_uri string, username string, password string) error { - log.Trace("Trying to setup new MongoClient") - var err error - if mongo_uri == "" { - log.Error("mongo uri not valid, because empty") - return errors.New("database name for NewMongoCollection not valid, because empty") - } - - if username == "" || password == "" { - log.Error("you must set your 'username' or 'password'") - return errors.New("username or password not correctly set") - } - - clientOpts := options.Client().ApplyURI(mongo_uri).SetAuth(options.Credential{ - Username: username, - Password: password, - }, - ) - - newContext, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - mongoclient, err = mongo.Connect(newContext, clientOpts) - const error_msg = "can't verify a connection" - if err != nil { - log.Error(error_msg) - return err - } - - err = mongoclient.Ping(context.Background(), nil) - - if err != nil { - log.Error(error_msg) - return err - } - - log.Trace("Looks like we succesfully setup a Mongodb client!") - return nil -} - -func CloseMongoDB() error { - err := mongoclient.Disconnect(context.Background()) - if err != nil { - log.Fatal("Failed to close mongoDB connection", err.Error()) - return err - } - return nil -} diff --git a/internal/database/playbook/playbook.go b/internal/database/playbook/playbook.go deleted file mode 100644 index be6ce99f..00000000 --- a/internal/database/playbook/playbook.go +++ /dev/null @@ -1,134 +0,0 @@ -package playbookrepository - -import ( - "errors" - - database "soarca/internal/database" - "soarca/internal/database/projections" - validator "soarca/internal/validators" - "soarca/pkg/models/api" - "soarca/pkg/models/cacao" - "soarca/pkg/models/decoder" -) - -type IPlaybookRepository interface { - GetPlaybooks() ([]cacao.Playbook, error) - GetPlaybookMetas() ([]api.PlaybookMeta, error) - Create(jsonData *[]byte) (cacao.Playbook, error) - Read(id string) (cacao.Playbook, error) - Update(id string, jsonData *[]byte) (cacao.Playbook, error) - Delete(id string) error -} - -type PlaybookRepository struct { - db database.Database - options database.FindOptions -} - -func SetupPlaybookRepository(db database.Database, options database.FindOptions) *PlaybookRepository { - return &PlaybookRepository{db: db, options: options} -} - -func (playbookRepo *PlaybookRepository) GetPlaybookMetas() ([]api.PlaybookMeta, error) { - playbookMetas, err := playbookRepo.db.Find(nil, playbookRepo.options.GetProjectionByType(projections.Meta)) - if err != nil { - return nil, err - } - - var returnPlaybookMetas []api.PlaybookMeta - - for _, playbookMeta := range playbookMetas { - playbookMeta, ok := playbookMeta.(cacao.Playbook) - if !ok { - return nil, errors.New("type assertion failed for cacao.Playbook to cacao.PlaybookMeta type") - } - returnPlaybookMetas = append(returnPlaybookMetas, api.PlaybookMeta{ - ID: playbookMeta.ID, - Name: playbookMeta.Name, - Description: playbookMeta.Description, - ValidFrom: playbookMeta.ValidFrom, - ValidUntil: playbookMeta.ValidUntil, - Labels: playbookMeta.Labels, - }) - } - return returnPlaybookMetas, nil -} - -func (playbookRepo *PlaybookRepository) GetPlaybooks() ([]cacao.Playbook, error) { - playbooks, err := playbookRepo.db.Find(nil) - if err != nil { - return nil, err - } - - var returnListPlaybooks []cacao.Playbook - for _, playbook := range playbooks { - // get the cacao playbook id and add to the return list - playbook, ok := playbook.(cacao.Playbook) - if !ok { - return nil, errors.New("type assertion failed for cacao.playbook type") - } - decode(&playbook) - returnListPlaybooks = append(returnListPlaybooks, playbook) - } - return returnListPlaybooks, nil -} - -func (playbookRepo *PlaybookRepository) Create(jsonData *[]byte) (cacao.Playbook, error) { - // validate the input object to required type and unmarshal - client_data, err := validator.UnmarshalJson[cacao.Playbook](jsonData) - if err != nil { - return cacao.Playbook{}, err - } - playbook, ok := client_data.(cacao.Playbook) - if !ok { - // handle incorrect casting - return cacao.Playbook{}, errors.New("failed to cast playbook object") - } - decode(&playbook) - - return playbook, playbookRepo.db.Create(client_data) -} - -func (playbookRepo *PlaybookRepository) Read(id string) (cacao.Playbook, error) { - returnedObject, err := playbookRepo.db.Read(id) - if err != nil { - return cacao.Playbook{}, err - } - - cacaoPlaybook, ok := returnedObject.(cacao.Playbook) - - if !ok { - err = errors.New("could not cast lookup object to cacao.Playbook type") - return cacao.Playbook{}, err - } - decode(&cacaoPlaybook) - - return cacaoPlaybook, nil -} - -func (playbookRepo *PlaybookRepository) Update(id string, jsonData *[]byte) (cacao.Playbook, error) { - // validate the input object to required type and unmarshal - client_data, err := validator.UnmarshalJson[cacao.Playbook](jsonData) - if err != nil { - return cacao.Playbook{}, err - } - cacaoPlaybook, ok := client_data.(cacao.Playbook) - if !ok { - err = errors.New("could not cast lookup object to cacao.Playbook type") - return cacao.Playbook{}, err - } - decode(&cacaoPlaybook) - return cacaoPlaybook, playbookRepo.db.Update(id, client_data) -} - -func (playbookRepo *PlaybookRepository) Delete(id string) error { - // validate the input object to required type and unmarshal - return playbookRepo.db.Delete(id) -} - -func decode(playbook *cacao.Playbook) { - if playbook.PlaybookVariables == nil { - playbook.PlaybookVariables = cacao.NewVariables() - } - decoder.SetPlaybookKeysAsId(playbook) -} diff --git a/internal/database/projections/projections.go b/internal/database/projections/projections.go deleted file mode 100644 index 3d75abb3..00000000 --- a/internal/database/projections/projections.go +++ /dev/null @@ -1,28 +0,0 @@ -package projections - -import "go.mongodb.org/mongo-driver/bson" - -type Projection uint8 - -const ( - Id Projection = iota - Meta -) - -func (p Projection) GetProjection() bson.D { - switch p { - case Id: - return bson.D{{Key: "_id", Value: 1}} - case Meta: - return bson.D{ - {Key: "_id", Value: 1}, - {Key: "name", Value: 1}, - {Key: "description", Value: 1}, - {Key: "created", Value: 1}, - {Key: "valid_from", Value: 1}, - {Key: "valid_until", Value: 1}, - {Key: "labels", Value: 1}, - } - } - return bson.D{} -} diff --git a/internal/database/projections/projections_test.go b/internal/database/projections/projections_test.go deleted file mode 100644 index 315244a2..00000000 --- a/internal/database/projections/projections_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package projections - -import ( - "testing" - - "github.com/go-playground/assert/v2" - "go.mongodb.org/mongo-driver/bson" -) - -func TestProjectionMeta(t *testing.T) { - testMeta := bson.D{ - {Key: "_id", Value: 1}, - {Key: "name", Value: 1}, - {Key: "description", Value: 1}, - {Key: "created", Value: 1}, - {Key: "valid_from", Value: 1}, - {Key: "valid_until", Value: 1}, - {Key: "labels", Value: 1}, - } - - validationMeta := Meta.GetProjection() - assert.Equal(t, testMeta, validationMeta) -} - -func TestProjectionID(t *testing.T) { - testMeta := bson.D{ - {Key: "_id", Value: 1}, - } - - validationMeta := Id.GetProjection() - assert.Equal(t, testMeta, validationMeta) -} diff --git a/internal/fins/registry.go b/internal/fins/registry.go new file mode 100644 index 00000000..48bea331 --- /dev/null +++ b/internal/fins/registry.go @@ -0,0 +1,141 @@ +package fin + +import ( + "context" + "reflect" + "time" + + "soarca/internal/logger" + "soarca/internal/store" + "soarca/internal/workflow/capability/fin/token" + "soarca/pkg/fins/protocol" + "soarca/pkg/utils/guid" +) + +var log *logger.Log + +func init() { + log = logger.Logger(reflect.TypeOf(struct{}{}).PkgPath(), logger.Info, "", logger.Json) +} + +// Registry implements the services.FinRegistry interface. +type Registry struct { + store storage.FinStore + config RegistryConfig + guid guid.IGuid +} + +// RegistryConfig holds configuration for the FIN registry service. +type RegistryConfig struct { + RegistrationToken string + StaleAfter time.Duration +} + +// NewRegistry creates a new FinRegistry service. +func NewRegistry(store storage.FinStore, config RegistryConfig, guid guid.IGuid) *Registry { + return &Registry{ + store: store, + config: config, + guid: guid, + } +} + +// RegisterFin registers a new FIN with the given capabilities. +func (r *Registry) RegisterFin(ctx context.Context, req fin.RegisterRequest) (finID string, finToken string, err error) { + if r.config.RegistrationToken == "" { + return "", "", fin.ErrRegistrationDisabled{} + } + if !token.Equal(req.RegistrationToken, r.config.RegistrationToken) { + return "", "", fin.ErrRegistrationTokenInvalid{} + } + if len(req.Capabilities) == 0 { + return "", "", fin.ErrNoCapabilities{} + } + for _, capability := range req.Capabilities { + if capability.Type == "" { + return "", "", fin.ErrCapabilityTypeEmpty{} + } + } + + finToken, err = token.Generate() + if err != nil { + return "", "", err + } + + record := fin.Record{ + FinId: r.guid.New().String(), + FinTokenHash: token.Hash(finToken), + DisplayName: req.DisplayName, + ProtocolVersion: req.ProtocolVersion, + Capabilities: req.Capabilities, + RegisteredAt: time.Now(), + LastSeen: time.Now(), + } + + if err := r.store.Create(ctx, record); err != nil { + return "", "", err + } + + log.Info("registered fin ", record.FinId, " (", record.DisplayName, ") with capabilities ", capabilityTypes(record.Capabilities)) + return record.FinId, finToken, nil +} + +// UnregisterFin unregisters a FIN by its token. +func (r *Registry) UnregisterFin(ctx context.Context, finToken string) error { + record, err := r.store.GetByTokenHash(ctx, token.Hash(finToken)) + if err != nil { + return err + } + return r.store.Delete(ctx, record.FinId) +} + +// ListFins returns all registered FINs with staleness information. +func (r *Registry) ListFins(ctx context.Context) ([]fin.Record, error) { + records, err := r.store.List(ctx) + if err != nil { + return nil, err + } + for i := range records { + records[i].Stale = r.isStale(records[i]) + } + return records, nil +} + +// GetFin retrieves a specific FIN record by ID. +func (r *Registry) GetFin(ctx context.Context, finID string) (fin.Record, error) { + record, err := r.store.Get(ctx, finID) + if err != nil { + return fin.Record{}, err + } + record.Stale = r.isStale(record) + return record, nil +} + +// DeleteFin removes a FIN record by ID (admin use). +func (r *Registry) DeleteFin(ctx context.Context, finID string) error { + return r.store.Delete(ctx, finID) +} + +// ValidateToken checks if a token is valid and returns the associated FIN record. +// Returns an error if the token is not found or invalid. +func (r *Registry) ValidateToken(ctx context.Context, finToken string) (string, error) { + record, err := r.store.GetByTokenHash(ctx, token.Hash(finToken)) + if err != nil { + return "", err + } + return record.FinId, nil +} + +// isStale checks if a FIN has exceeded the staleness threshold. +func (r *Registry) isStale(record fin.Record) bool { + return time.Since(record.LastSeen) > r.config.StaleAfter +} + +// capabilityTypes extracts capability type strings from capability records. +func capabilityTypes(capabilities []fin.Capability) []string { + types := make([]string, 0, len(capabilities)) + for _, capability := range capabilities { + types = append(types, capability.Type) + } + return types +} diff --git a/internal/fins/work.go b/internal/fins/work.go new file mode 100644 index 00000000..a90ca7a9 --- /dev/null +++ b/internal/fins/work.go @@ -0,0 +1,107 @@ +package fin + +import ( + "context" + "time" + + "github.com/google/uuid" + + "soarca/internal/store" + "soarca/internal/workflow/capability/fin/queue" + "soarca/internal/workflow/capability/fin/token" + "soarca/pkg/fins/protocol" +) + +// WorkService implements the services.FinWorkService interface. +// It manages leased FIN work items, handling polling, result submission, and heartbeats. +type WorkService struct { + store storage.FinStore + queue *queue.Queue + config WorkServiceConfig +} + +// WorkServiceConfig holds configuration for the FIN work service. +type WorkServiceConfig struct { + LongPollTimeoutSeconds int + JobLeaseSeconds int +} + +// NewWorkService creates a new FinWorkService. +func NewWorkService(store storage.FinStore, q *queue.Queue, config WorkServiceConfig) *WorkService { + return &WorkService{ + store: store, + queue: q, + config: config, + } +} + +// PollJob polls for available work matching the FIN's capabilities. +// Updates the FIN's last-seen timestamp. +func (w *WorkService) PollJob(ctx context.Context, finToken string, pollReq fin.PollRequest) (*fin.Job, error) { + record, err := w.store.GetByTokenHash(ctx, token.Hash(finToken)) + if err != nil { + return nil, err + } + + // Update last-seen timestamp (touch the FIN). + if err := w.store.Touch(ctx, record.FinId, time.Now()); err != nil { + log.Warning("failed to update last-seen for fin ", record.FinId, ": ", err) + } + + // Calculate poll timeout. + timeout := time.Duration(w.config.LongPollTimeoutSeconds) * time.Second + if w.config.LongPollTimeoutSeconds <= 0 { + timeout = 25 * time.Second + } + pollCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + // Claim a job matching the FIN's capabilities. + job, err := w.queue.Claim(pollCtx, capabilityTypes(record.Capabilities), record.FinId) + if err != nil { + return nil, err + } + return &job, nil +} + +// SubmitJobResult submits the result of a claimed job. +// Updates the FIN's last-seen timestamp. +func (w *WorkService) SubmitJobResult(ctx context.Context, finToken string, jobID uuid.UUID, result fin.JobResult) error { + record, err := w.store.GetByTokenHash(ctx, token.Hash(finToken)) + if err != nil { + return err + } + + // Submit the result to the queue. + if err := w.queue.Submit(jobID, record.FinId, result); err != nil { + return err + } + + // Update last-seen timestamp. + if err := w.store.Touch(ctx, record.FinId, time.Now()); err != nil { + log.Warning("failed to update last-seen for fin ", record.FinId, ": ", err) + } + + return nil +} + +// HeartbeatJob extends the lease on an in-flight job (status-ping). +// Updates the FIN's last-seen timestamp. +func (w *WorkService) HeartbeatJob(ctx context.Context, finToken string, jobID uuid.UUID) error { + record, err := w.store.GetByTokenHash(ctx, token.Hash(finToken)) + if err != nil { + return err + } + + // Extend the lease. + if err := w.queue.ExtendLease(jobID, record.FinId, w.config.JobLeaseSeconds); err != nil { + return err + } + + // Update last-seen timestamp. + if err := w.store.Touch(ctx, record.FinId, time.Now()); err != nil { + log.Warning("failed to update last-seen for fin ", record.FinId, ": ", err) + } + + return nil +} diff --git a/internal/logger/logger.go b/internal/logger/log.go similarity index 100% rename from internal/logger/logger.go rename to internal/logger/log.go diff --git a/internal/manual/inbox.go b/internal/manual/inbox.go new file mode 100644 index 00000000..37e523a4 --- /dev/null +++ b/internal/manual/inbox.go @@ -0,0 +1,33 @@ +package manual + +import ( + "soarca/internal/workflow/capability/manual/inbox" + "soarca/internal/manual/model" + "soarca/internal/runs/model" +) + +// Inbox implements the services.ManualInbox interface. +// It delegates manual manual inbox storage operations to the manual inbox. +type Inbox struct { + storage inbox.Store +} + +// NewInbox creates a manual inbox service. +func NewInbox(storage inbox.Store) *Inbox { + return &Inbox{storage: storage} +} + +// ListPendingCommands returns all pending manual commands. +func (i *Inbox) ListPendingCommands() ([]manual.CommandInfo, error) { + return i.storage.GetPendingCommands() +} + +// GetPendingCommand returns one pending manual command. +func (i *Inbox) GetPendingCommand(metadata run.Metadata) (manual.CommandInfo, error) { + return i.storage.GetPendingCommand(metadata) +} + +// ContinuePendingCommand submits a response for a pending manual command. +func (i *Inbox) ContinuePendingCommand(response manual.Response) error { + return i.storage.PostContinue(response) +} diff --git a/pkg/models/manual/manual.go b/internal/manual/model/command.go similarity index 62% rename from pkg/models/manual/manual.go rename to internal/manual/model/command.go index 638a0cea..53662c81 100644 --- a/pkg/models/manual/manual.go +++ b/internal/manual/model/command.go @@ -2,9 +2,9 @@ package manual import ( "context" - "soarca/pkg/core/capability" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" + "soarca/internal/workflow/capability" + "soarca/pkg/cacao" + "soarca/internal/runs/model" ) // ################################################################################ @@ -18,36 +18,36 @@ const ( ManualResponseFailureStatus ManualResponseStatus = "failure" ) -type InteractionStorageEntry struct { +type PendingCommand struct { CommandInfo CommandInfo - Channel chan InteractionResponse + Channel chan Response } -// Object passed by the manual capability to the Interaction module +// Object passed by the manual capability to the manual inbox type CommandInfo struct { - Metadata execution.Metadata + Metadata run.Metadata Context capability.Context OutArgsVariables cacao.Variables } -// Deep copy for the command that the Interaction module notifies to the integrations -type InteractionIntegrationCommand struct { - Metadata execution.Metadata +// Deep copy for the command that the manual inbox notifies to the integrations +type Notification struct { + Metadata run.Metadata Context capability.Context OutArgsVariables cacao.Variables } -// Object returned to the Interaction object in fulfilment of a manual command -type InteractionResponse struct { - Metadata execution.Metadata +// Object returned to the manual inbox in fulfilment of a manual command +type Response struct { + Metadata run.Metadata ResponseStatus ManualResponseStatus ResponseError error OutArgsVariables cacao.Variables } -type ManualCapabilityCommunication struct { +type Waiter struct { TimeoutContext context.Context - Channel chan InteractionResponse + Channel chan Response } // Errors ##################################################################### diff --git a/internal/orchestrator/contracts.go b/internal/orchestrator/contracts.go new file mode 100644 index 00000000..b49be930 --- /dev/null +++ b/internal/orchestrator/contracts.go @@ -0,0 +1,133 @@ +package orchestrator + +import ( + "context" + + "soarca/internal/playbooks" + "soarca/pkg/cacao" + "soarca/pkg/fins/protocol" + "soarca/internal/manual/model" + "soarca/internal/runs/model" + + "github.com/google/uuid" +) + +// ============================================================================ +// OPERATIONAL SERVICES +// ============================================================================ + +// FinRegistry manages FIN registration and admin lifecycle. +// +// Dependencies: FinStore (only) +// Does NOT depend on: playbook run, job leasing. +type FinRegistry interface { + // RegisterFin registers a new FIN with the given capabilities. + RegisterFin(ctx context.Context, req fin.RegisterRequest) (finID string, token string, err error) + + // UnregisterFin unregisters a FIN and cleans up its record. + UnregisterFin(ctx context.Context, finToken string) error + + // ListFins returns all registered FINs (admin use). + ListFins(ctx context.Context) (fins []fin.Record, err error) + + // GetFin retrieves a specific FIN record by ID (admin use). + GetFin(ctx context.Context, finID string) (record fin.Record, err error) + + // DeleteFin removes a FIN record (admin use). + DeleteFin(ctx context.Context, finID string) error + + // ValidateToken checks if a token is valid and returns the FIN ID. + // Used by auth middleware to verify credentials. + ValidateToken(ctx context.Context, finToken string) (finID string, err error) +} + +// FinWorkService manages leased FIN work items. +// +// Dependencies: FinQueue, FinStore (only) +// Does NOT depend on: playbook run, manual commands, HTTP. +// +// Lease semantics belong here, not in the run runtime. +type FinWorkService interface { + // PollJob polls for available work matching FIN's capabilities. + // Long-polls until a job is available or context timeout/cancellation. + PollJob(ctx context.Context, finToken string, pollReq fin.PollRequest) (job *fin.Job, err error) + + // SubmitJobResult submits the result of a claimed job. + SubmitJobResult(ctx context.Context, finToken string, jobID uuid.UUID, result fin.JobResult) error + + // HeartbeatJob extends the lease on an in-flight job (status-ping). + HeartbeatJob(ctx context.Context, finToken string, jobID uuid.UUID) error +} + +// ManualInbox manages manual step resolution during playbook run. +// Depends on: manual inbox +// Does NOT depend on: FIN leasing or claim semantics. +// +// Responsibility: tracking pending manual commands, allowing operators to +// view pending steps, and providing responses for outstanding manual work. +type ManualInbox interface { + // ListPendingCommands returns all pending manual steps across all runs. + ListPendingCommands() (commands []manual.CommandInfo, err error) + + // GetPendingCommand retrieves a specific pending manual step. + GetPendingCommand(metadata run.Metadata) (command manual.CommandInfo, err error) + + // ContinuePendingCommand resolves a pending manual step with the operator's response. + ContinuePendingCommand(response manual.Response) error +} + +// ============================================================================ +// APPLICATION SERVICES (thin orchestrators) +// ============================================================================ + +// PlaybookService provides CRUD operations over the playbook repository. +// Depends on: PlaybookStore +// +// Responsibility: playbook lifecycle (create, read, update, delete, list). +// Future: could add versioning, validation, domain rules. +type PlaybookService interface { + // ListPlaybooks returns all stored playbooks. + ListPlaybooks(ctx context.Context) (playbooks []cacao.Playbook, err error) + + // GetPlaybook retrieves a playbook by ID. + GetPlaybook(ctx context.Context, playbookID string) (playbook *cacao.Playbook, err error) + + // CreatePlaybook stores a new playbook. + CreatePlaybook(ctx context.Context, playbook *cacao.Playbook) error + + // UpdatePlaybook replaces an existing playbook. + UpdatePlaybook(ctx context.Context, playbookID string, playbook *cacao.Playbook) error + + // DeletePlaybook removes a playbook. + DeletePlaybook(ctx context.Context, playbookID string) error + + // ListPlaybookMetas returns metadata for all playbooks (efficient list). + ListPlaybookMetas(ctx context.Context) (metas []playbooks.Meta, err error) +} + +// ============================================================================ +// DEPENDENCY NOTES +// ============================================================================ + +/* +Dependency Flow (what depends on what): + + runs.Runner (core kernel) + └─ owns: engine (decomposer factory), PlaybookStore, RunState + + FinRegistry (independent) + └─ owns: FinStore + + FinWorkService (leased work) + └─ owns: FinQueue, FinStore + + ManualInbox + └─ depends on: manual inbox + + PlaybookService (CRUD adapter) + └─ depends on: PlaybookStore + +HTTP Handlers (thin transport) + └─ depend on: these services + └─ do NOT depend on: stores, queues, runtime directly +*/ diff --git a/internal/orchestrator/orchestrator.go b/internal/orchestrator/orchestrator.go new file mode 100644 index 00000000..4564bfe2 --- /dev/null +++ b/internal/orchestrator/orchestrator.go @@ -0,0 +1,158 @@ +package orchestrator + +import ( + "context" + "fmt" + "reflect" + + "soarca/internal/config" + fins "soarca/internal/fins" + "soarca/internal/logger" + manualsvc "soarca/internal/manual" + playbookservice "soarca/internal/playbooks/library" + "soarca/internal/reporting/reporter/downstream_reporter/runstate" + "soarca/internal/runs" + "soarca/internal/runs/engine" + storage "soarca/internal/store" + storagesql "soarca/internal/store/sql" + "soarca/internal/workflow/capability/fin/queue" + "soarca/internal/workflow/capability/manual/inbox" + "soarca/pkg/utils/guid" + timeutil "soarca/pkg/utils/time" +) + +var log *logger.Log + +func init() { + log = logger.Logger(reflect.TypeOf(Runtime{}).PkgPath(), logger.Info, "", logger.Json) +} + +// Options contains all configuration needed to build the runtime and its orchestrator. +type Options struct { + Storage config.StorageConfig + RunState config.RunStateConfig + Fin config.FinConfig + HTTP config.HTTPConfig + TheHive config.TheHiveConfig +} + +// Operations is the use case surface the runtime offers to any driver +// (HTTP, gRPC, CLI, embedded SDK). It carries behaviour only: no +// infrastructure and no getters to reach through. Drivers hold this value, +// never the Runtime itself. +type Operations struct { + Playbooks PlaybookService + Runs runs.Runner + Fins FinRegistry + Work FinWorkService + Manual ManualInbox +} + +// Runtime owns construction and lifetime of the orchestrator's dependencies. +type Runtime struct { + // Core infrastructure (shared across services) + store storage.Store + playbookStore storage.PlaybookStore + finStore storage.FinStore + runState *runstate.RunState + manual *inbox.Inbox + finQueue *queue.Queue + + // Application services + runs runs.Runner + finRegistry FinRegistry + finWorkService FinWorkService + manualInbox ManualInbox + playbookService PlaybookService +} + +// New creates and initializes application dependencies and orchestrator. +func New(opts Options) (*Runtime, error) { + runtime := &Runtime{} + + // Initialize storage layer + if err := runtime.initializeStorage(opts.Storage); err != nil { + return nil, fmt.Errorf("failed to initialize storage: %w", err) + } + + // Initialize shared infrastructure + runtime.runState = runstate.New(&timeutil.Time{}, opts.RunState.MaxRuns) + runtime.manual = inbox.New([]inbox.Notifier{}) + runtime.finQueue = queue.New() + + // Create FIN services + runtime.finRegistry = fins.NewRegistry( + runtime.finStore, + fins.RegistryConfig{ + RegistrationToken: opts.Fin.RegistrationToken, + StaleAfter: opts.Fin.StaleAfter, + }, + new(guid.Guid), + ) + + runtime.finWorkService = fins.NewWorkService( + runtime.finStore, + runtime.finQueue, + fins.WorkServiceConfig{ + LongPollTimeoutSeconds: opts.Fin.LongPollTimeoutSeconds, + JobLeaseSeconds: opts.Fin.JobLeaseSeconds, + }, + ) + + // Create manual service + runtime.manualInbox = manualsvc.NewInbox(runtime.manual) + + // Create playbook service + runtime.playbookService = playbookservice.New(runtime.playbookStore) + + // Create the run engine and the run service that drives it. + runEngine := engine.New(engine.Deps{ + ManualInbox: runtime.manual, + RunState: runtime.runState, + FinQueue: runtime.finQueue, + FinStore: runtime.finStore, + PlaybookStore: runtime.playbookStore, + SkipCertValidation: opts.HTTP.SkipCertValidation, + FinStaleAfter: opts.Fin.StaleAfter, + TheHive: opts.TheHive, + }) + runtime.runs = runs.New(runEngine.NewWalker, runtime.playbookStore, runtime.runState) + + return runtime, nil +} + +// initializeStorage opens the SQL store and applies migrations. +func (r *Runtime) initializeStorage(storageCfg config.StorageConfig) error { + log.Infof("Initializing storage from %s", storageCfg.DatabaseURL) + store, err := storagesql.New(context.Background(), storageCfg.DatabaseURL) + if err != nil { + return err + } + + r.store = store + r.playbookStore = store.Playbooks() + r.finStore = store.Fins() + return nil +} + +// Close releases any resources held by the app. +func (r *Runtime) Close() error { + if r.finQueue != nil { + r.finQueue.Close() + } + if r.store != nil { + return r.store.Close(context.Background()) + } + return nil +} + +// Operations returns the use case surface for drivers. +func (r *Runtime) Operations() Operations { + return Operations{ + Playbooks: r.playbookService, + Runs: r.runs, + Fins: r.finRegistry, + Work: r.finWorkService, + Manual: r.manualInbox, + } +} diff --git a/internal/orchestrator/runtime_test.go b/internal/orchestrator/runtime_test.go new file mode 100644 index 00000000..b7dc1ebb --- /dev/null +++ b/internal/orchestrator/runtime_test.go @@ -0,0 +1,44 @@ +package orchestrator + +import ( + "testing" + + "soarca/internal/config" +) + +func TestNewInitializesDependencies(t *testing.T) { + runtime, err := New(Options{ + Storage: config.StorageConfig{ + DatabaseURL: "sqlite://:memory:", + }, + RunState: config.RunStateConfig{ + MaxRuns: 3, + }, + }) + if err != nil { + t.Fatalf("New() returned error: %v", err) + } + t.Cleanup(func() { + if err := runtime.Close(); err != nil { + t.Fatalf("Close() returned error: %v", err) + } + }) + + ops := runtime.Operations() + + if ops.Playbooks == nil { + t.Fatal("Operations.Playbooks is nil") + } + if ops.Runs == nil { + t.Fatal("Operations.Runs is nil") + } + if ops.Fins == nil { + t.Fatal("Operations.Fins is nil") + } + if ops.Work == nil { + t.Fatal("Operations.Work is nil") + } + if ops.Manual == nil { + t.Fatal("Operations.Manual is nil") + } +} diff --git a/pkg/models/decoder/decoder.go b/internal/playbooks/decoder/decode.go similarity index 96% rename from pkg/models/decoder/decoder.go rename to internal/playbooks/decoder/decode.go index af13b601..851e8bea 100644 --- a/pkg/models/decoder/decoder.go +++ b/internal/playbooks/decoder/decode.go @@ -4,8 +4,8 @@ import ( "encoding/json" "reflect" "soarca/internal/logger" - "soarca/pkg/models/cacao" - "soarca/pkg/models/validator" + "soarca/pkg/cacao" + "soarca/internal/playbooks/validator" ) type Empty struct{} diff --git a/pkg/models/decoder/decoder_test.go b/internal/playbooks/decoder/decoder_test.go similarity index 100% rename from pkg/models/decoder/decoder_test.go rename to internal/playbooks/decoder/decoder_test.go diff --git a/internal/playbooks/library/service.go b/internal/playbooks/library/service.go new file mode 100644 index 00000000..6d30ed39 --- /dev/null +++ b/internal/playbooks/library/service.go @@ -0,0 +1,54 @@ +package playbook + +import ( + "context" + + "soarca/internal/store" + "soarca/internal/playbooks" + "soarca/pkg/cacao" +) + +// Service implements the services.PlaybookService interface. +type Service struct { + store storage.PlaybookStore +} + +// New creates a new playbook service. +func New(store storage.PlaybookStore) *Service { + return &Service{store: store} +} + +// ListPlaybooks returns all stored playbooks. +func (s *Service) ListPlaybooks(ctx context.Context) ([]cacao.Playbook, error) { + return s.store.List(ctx) +} + +// GetPlaybook retrieves a playbook by ID. +func (s *Service) GetPlaybook(ctx context.Context, playbookID string) (*cacao.Playbook, error) { + playbook, err := s.store.Get(ctx, playbookID) + if err != nil { + return nil, err + } + return &playbook, nil +} + +// CreatePlaybook stores a new playbook. +func (s *Service) CreatePlaybook(ctx context.Context, playbook *cacao.Playbook) error { + return s.store.Create(ctx, *playbook) +} + +// UpdatePlaybook replaces an existing playbook. +func (s *Service) UpdatePlaybook(ctx context.Context, playbookID string, playbook *cacao.Playbook) error { + playbook.ID = playbookID + return s.store.Update(ctx, *playbook) +} + +// DeletePlaybook removes a playbook. +func (s *Service) DeletePlaybook(ctx context.Context, playbookID string) error { + return s.store.Delete(ctx, playbookID) +} + +// ListPlaybookMetas returns playbook metadata entries. +func (s *Service) ListPlaybookMetas(ctx context.Context) ([]playbooks.Meta, error) { + return s.store.ListMeta(ctx) +} diff --git a/internal/playbooks/meta.go b/internal/playbooks/meta.go new file mode 100644 index 00000000..3436c8da --- /dev/null +++ b/internal/playbooks/meta.go @@ -0,0 +1,14 @@ +package playbooks +package playbooks + +import "time" + +// Meta is the lightweight summary used when listing stored playbooks. +type Meta struct { + ID string + Name string + Description string + ValidFrom time.Time + ValidUntil time.Time + Labels []string +} \ No newline at end of file diff --git a/pkg/models/validator/playbook.go b/internal/playbooks/validator/playbook.go similarity index 98% rename from pkg/models/validator/playbook.go rename to internal/playbooks/validator/playbook.go index a704ab13..984e2dae 100644 --- a/pkg/models/validator/playbook.go +++ b/internal/playbooks/validator/playbook.go @@ -6,7 +6,7 @@ import ( "net/mail" "soarca/internal/logger" - "soarca/pkg/models/cacao" + "soarca/pkg/cacao" ) func init() { @@ -47,7 +47,7 @@ func IsSafeCacaoWorkflow(playbook *cacao.Playbook) error { return nil } -// Given one step id, checks if its execution is safe +// Given one step id, checks if its run is safe // with respect to all properties being present in the playbook. // All checks are in O(1) as it's all dictionary key lookups func isSafeWorkflowStep(playbook *cacao.Playbook, stepId string) error { @@ -169,7 +169,7 @@ func checkAllWorkflowBranchesEnd(playbook *cacao.Playbook) error { } // Navigates a CACAO workflow object recursively as depth-first tree -// on possible branches of the workflow execution. +// on possible branches of the workflow run. // // branchSequence parameter "branching sequence" collects all the steps ID // in the current branch and copies its values to diff --git a/pkg/models/validator/schema.go b/internal/playbooks/validator/schema.go similarity index 99% rename from pkg/models/validator/schema.go rename to internal/playbooks/validator/schema.go index 5e955482..bbb12cad 100644 --- a/pkg/models/validator/schema.go +++ b/internal/playbooks/validator/schema.go @@ -7,7 +7,7 @@ import ( "io/fs" "reflect" "soarca/internal/logger" - "soarca/pkg/models/cacao" + "soarca/pkg/cacao" "soarca/pkg/utils" "strings" diff --git a/pkg/models/validator/schemas/agent-target/agent-target.json b/internal/playbooks/validator/schemas/agent-target/agent-target.json similarity index 100% rename from pkg/models/validator/schemas/agent-target/agent-target.json rename to internal/playbooks/validator/schemas/agent-target/agent-target.json diff --git a/pkg/models/validator/schemas/agent-target/group.json b/internal/playbooks/validator/schemas/agent-target/group.json similarity index 100% rename from pkg/models/validator/schemas/agent-target/group.json rename to internal/playbooks/validator/schemas/agent-target/group.json diff --git a/pkg/models/validator/schemas/agent-target/http-api.json b/internal/playbooks/validator/schemas/agent-target/http-api.json similarity index 100% rename from pkg/models/validator/schemas/agent-target/http-api.json rename to internal/playbooks/validator/schemas/agent-target/http-api.json diff --git a/pkg/models/validator/schemas/agent-target/individual.json b/internal/playbooks/validator/schemas/agent-target/individual.json similarity index 100% rename from pkg/models/validator/schemas/agent-target/individual.json rename to internal/playbooks/validator/schemas/agent-target/individual.json diff --git a/pkg/models/validator/schemas/agent-target/linux.json b/internal/playbooks/validator/schemas/agent-target/linux.json similarity index 100% rename from pkg/models/validator/schemas/agent-target/linux.json rename to internal/playbooks/validator/schemas/agent-target/linux.json diff --git a/pkg/models/validator/schemas/agent-target/location.json b/internal/playbooks/validator/schemas/agent-target/location.json similarity index 100% rename from pkg/models/validator/schemas/agent-target/location.json rename to internal/playbooks/validator/schemas/agent-target/location.json diff --git a/pkg/models/validator/schemas/agent-target/net-address.json b/internal/playbooks/validator/schemas/agent-target/net-address.json similarity index 100% rename from pkg/models/validator/schemas/agent-target/net-address.json rename to internal/playbooks/validator/schemas/agent-target/net-address.json diff --git a/pkg/models/validator/schemas/agent-target/organization.json b/internal/playbooks/validator/schemas/agent-target/organization.json similarity index 100% rename from pkg/models/validator/schemas/agent-target/organization.json rename to internal/playbooks/validator/schemas/agent-target/organization.json diff --git a/pkg/models/validator/schemas/agent-target/sector.json b/internal/playbooks/validator/schemas/agent-target/sector.json similarity index 100% rename from pkg/models/validator/schemas/agent-target/sector.json rename to internal/playbooks/validator/schemas/agent-target/sector.json diff --git a/pkg/models/validator/schemas/agent-target/security-category.json b/internal/playbooks/validator/schemas/agent-target/security-category.json similarity index 100% rename from pkg/models/validator/schemas/agent-target/security-category.json rename to internal/playbooks/validator/schemas/agent-target/security-category.json diff --git a/pkg/models/validator/schemas/agent-target/ssh.json b/internal/playbooks/validator/schemas/agent-target/ssh.json similarity index 100% rename from pkg/models/validator/schemas/agent-target/ssh.json rename to internal/playbooks/validator/schemas/agent-target/ssh.json diff --git a/pkg/models/validator/schemas/authentication-info/authentication-info.json b/internal/playbooks/validator/schemas/authentication-info/authentication-info.json similarity index 100% rename from pkg/models/validator/schemas/authentication-info/authentication-info.json rename to internal/playbooks/validator/schemas/authentication-info/authentication-info.json diff --git a/pkg/models/validator/schemas/authentication-info/http-basic.json b/internal/playbooks/validator/schemas/authentication-info/http-basic.json similarity index 100% rename from pkg/models/validator/schemas/authentication-info/http-basic.json rename to internal/playbooks/validator/schemas/authentication-info/http-basic.json diff --git a/pkg/models/validator/schemas/authentication-info/oauth2.json b/internal/playbooks/validator/schemas/authentication-info/oauth2.json similarity index 100% rename from pkg/models/validator/schemas/authentication-info/oauth2.json rename to internal/playbooks/validator/schemas/authentication-info/oauth2.json diff --git a/pkg/models/validator/schemas/authentication-info/user-auth.json b/internal/playbooks/validator/schemas/authentication-info/user-auth.json similarity index 100% rename from pkg/models/validator/schemas/authentication-info/user-auth.json rename to internal/playbooks/validator/schemas/authentication-info/user-auth.json diff --git a/pkg/models/validator/schemas/commands/bash.json b/internal/playbooks/validator/schemas/commands/bash.json similarity index 100% rename from pkg/models/validator/schemas/commands/bash.json rename to internal/playbooks/validator/schemas/commands/bash.json diff --git a/pkg/models/validator/schemas/commands/caldera-cmd.json b/internal/playbooks/validator/schemas/commands/caldera-cmd.json similarity index 100% rename from pkg/models/validator/schemas/commands/caldera-cmd.json rename to internal/playbooks/validator/schemas/commands/caldera-cmd.json diff --git a/pkg/models/validator/schemas/commands/command-data.json b/internal/playbooks/validator/schemas/commands/command-data.json similarity index 100% rename from pkg/models/validator/schemas/commands/command-data.json rename to internal/playbooks/validator/schemas/commands/command-data.json diff --git a/pkg/models/validator/schemas/commands/elastic.json b/internal/playbooks/validator/schemas/commands/elastic.json similarity index 100% rename from pkg/models/validator/schemas/commands/elastic.json rename to internal/playbooks/validator/schemas/commands/elastic.json diff --git a/pkg/models/validator/schemas/commands/http-api.json b/internal/playbooks/validator/schemas/commands/http-api.json similarity index 100% rename from pkg/models/validator/schemas/commands/http-api.json rename to internal/playbooks/validator/schemas/commands/http-api.json diff --git a/pkg/models/validator/schemas/commands/jupyter.json b/internal/playbooks/validator/schemas/commands/jupyter.json similarity index 100% rename from pkg/models/validator/schemas/commands/jupyter.json rename to internal/playbooks/validator/schemas/commands/jupyter.json diff --git a/pkg/models/validator/schemas/commands/kestrel.json b/internal/playbooks/validator/schemas/commands/kestrel.json similarity index 100% rename from pkg/models/validator/schemas/commands/kestrel.json rename to internal/playbooks/validator/schemas/commands/kestrel.json diff --git a/pkg/models/validator/schemas/commands/manual.json b/internal/playbooks/validator/schemas/commands/manual.json similarity index 100% rename from pkg/models/validator/schemas/commands/manual.json rename to internal/playbooks/validator/schemas/commands/manual.json diff --git a/pkg/models/validator/schemas/commands/openc2-http.json b/internal/playbooks/validator/schemas/commands/openc2-http.json similarity index 100% rename from pkg/models/validator/schemas/commands/openc2-http.json rename to internal/playbooks/validator/schemas/commands/openc2-http.json diff --git a/pkg/models/validator/schemas/commands/powershell.json b/internal/playbooks/validator/schemas/commands/powershell.json similarity index 100% rename from pkg/models/validator/schemas/commands/powershell.json rename to internal/playbooks/validator/schemas/commands/powershell.json diff --git a/pkg/models/validator/schemas/commands/sigma.json b/internal/playbooks/validator/schemas/commands/sigma.json similarity index 100% rename from pkg/models/validator/schemas/commands/sigma.json rename to internal/playbooks/validator/schemas/commands/sigma.json diff --git a/pkg/models/validator/schemas/commands/ssh.json b/internal/playbooks/validator/schemas/commands/ssh.json similarity index 100% rename from pkg/models/validator/schemas/commands/ssh.json rename to internal/playbooks/validator/schemas/commands/ssh.json diff --git a/pkg/models/validator/schemas/commands/yara.json b/internal/playbooks/validator/schemas/commands/yara.json similarity index 100% rename from pkg/models/validator/schemas/commands/yara.json rename to internal/playbooks/validator/schemas/commands/yara.json diff --git a/pkg/models/validator/schemas/data-markings/data-marking.json b/internal/playbooks/validator/schemas/data-markings/data-marking.json similarity index 100% rename from pkg/models/validator/schemas/data-markings/data-marking.json rename to internal/playbooks/validator/schemas/data-markings/data-marking.json diff --git a/pkg/models/validator/schemas/data-markings/marking-iep.json b/internal/playbooks/validator/schemas/data-markings/marking-iep.json similarity index 100% rename from pkg/models/validator/schemas/data-markings/marking-iep.json rename to internal/playbooks/validator/schemas/data-markings/marking-iep.json diff --git a/pkg/models/validator/schemas/data-markings/marking-statement.json b/internal/playbooks/validator/schemas/data-markings/marking-statement.json similarity index 100% rename from pkg/models/validator/schemas/data-markings/marking-statement.json rename to internal/playbooks/validator/schemas/data-markings/marking-statement.json diff --git a/pkg/models/validator/schemas/data-markings/marking-tlp.json b/internal/playbooks/validator/schemas/data-markings/marking-tlp.json similarity index 100% rename from pkg/models/validator/schemas/data-markings/marking-tlp.json rename to internal/playbooks/validator/schemas/data-markings/marking-tlp.json diff --git a/pkg/models/validator/schemas/data-types/civic-location.json b/internal/playbooks/validator/schemas/data-types/civic-location.json similarity index 100% rename from pkg/models/validator/schemas/data-types/civic-location.json rename to internal/playbooks/validator/schemas/data-types/civic-location.json diff --git a/pkg/models/validator/schemas/data-types/contact.json b/internal/playbooks/validator/schemas/data-types/contact.json similarity index 100% rename from pkg/models/validator/schemas/data-types/contact.json rename to internal/playbooks/validator/schemas/data-types/contact.json diff --git a/pkg/models/validator/schemas/data-types/external-reference.json b/internal/playbooks/validator/schemas/data-types/external-reference.json similarity index 100% rename from pkg/models/validator/schemas/data-types/external-reference.json rename to internal/playbooks/validator/schemas/data-types/external-reference.json diff --git a/pkg/models/validator/schemas/data-types/identifier.json b/internal/playbooks/validator/schemas/data-types/identifier.json similarity index 100% rename from pkg/models/validator/schemas/data-types/identifier.json rename to internal/playbooks/validator/schemas/data-types/identifier.json diff --git a/pkg/models/validator/schemas/data-types/playbook-processing-summary.json b/internal/playbooks/validator/schemas/data-types/playbook-processing-summary.json similarity index 100% rename from pkg/models/validator/schemas/data-types/playbook-processing-summary.json rename to internal/playbooks/validator/schemas/data-types/playbook-processing-summary.json diff --git a/pkg/models/validator/schemas/data-types/signature.json b/internal/playbooks/validator/schemas/data-types/signature.json similarity index 100% rename from pkg/models/validator/schemas/data-types/signature.json rename to internal/playbooks/validator/schemas/data-types/signature.json diff --git a/pkg/models/validator/schemas/data-types/timestamp.json b/internal/playbooks/validator/schemas/data-types/timestamp.json similarity index 100% rename from pkg/models/validator/schemas/data-types/timestamp.json rename to internal/playbooks/validator/schemas/data-types/timestamp.json diff --git a/pkg/models/validator/schemas/data-types/variable.json b/internal/playbooks/validator/schemas/data-types/variable.json similarity index 100% rename from pkg/models/validator/schemas/data-types/variable.json rename to internal/playbooks/validator/schemas/data-types/variable.json diff --git a/pkg/models/validator/schemas/extension-definition/extension-definition.json b/internal/playbooks/validator/schemas/extension-definition/extension-definition.json similarity index 100% rename from pkg/models/validator/schemas/extension-definition/extension-definition.json rename to internal/playbooks/validator/schemas/extension-definition/extension-definition.json diff --git a/pkg/models/validator/schemas/playbook.json b/internal/playbooks/validator/schemas/playbook.json similarity index 100% rename from pkg/models/validator/schemas/playbook.json rename to internal/playbooks/validator/schemas/playbook.json diff --git a/pkg/models/validator/schemas/workflows/action.json b/internal/playbooks/validator/schemas/workflows/action.json similarity index 100% rename from pkg/models/validator/schemas/workflows/action.json rename to internal/playbooks/validator/schemas/workflows/action.json diff --git a/pkg/models/validator/schemas/workflows/end.json b/internal/playbooks/validator/schemas/workflows/end.json similarity index 100% rename from pkg/models/validator/schemas/workflows/end.json rename to internal/playbooks/validator/schemas/workflows/end.json diff --git a/pkg/models/validator/schemas/workflows/if-condition.json b/internal/playbooks/validator/schemas/workflows/if-condition.json similarity index 100% rename from pkg/models/validator/schemas/workflows/if-condition.json rename to internal/playbooks/validator/schemas/workflows/if-condition.json diff --git a/pkg/models/validator/schemas/workflows/parallel.json b/internal/playbooks/validator/schemas/workflows/parallel.json similarity index 100% rename from pkg/models/validator/schemas/workflows/parallel.json rename to internal/playbooks/validator/schemas/workflows/parallel.json diff --git a/pkg/models/validator/schemas/workflows/playbook-action.json b/internal/playbooks/validator/schemas/workflows/playbook-action.json similarity index 100% rename from pkg/models/validator/schemas/workflows/playbook-action.json rename to internal/playbooks/validator/schemas/workflows/playbook-action.json diff --git a/pkg/models/validator/schemas/workflows/start.json b/internal/playbooks/validator/schemas/workflows/start.json similarity index 100% rename from pkg/models/validator/schemas/workflows/start.json rename to internal/playbooks/validator/schemas/workflows/start.json diff --git a/pkg/models/validator/schemas/workflows/switch-condition.json b/internal/playbooks/validator/schemas/workflows/switch-condition.json similarity index 100% rename from pkg/models/validator/schemas/workflows/switch-condition.json rename to internal/playbooks/validator/schemas/workflows/switch-condition.json diff --git a/pkg/models/validator/schemas/workflows/while-condition.json b/internal/playbooks/validator/schemas/workflows/while-condition.json similarity index 100% rename from pkg/models/validator/schemas/workflows/while-condition.json rename to internal/playbooks/validator/schemas/workflows/while-condition.json diff --git a/pkg/models/validator/schemas/workflows/workflow-step.json b/internal/playbooks/validator/schemas/workflows/workflow-step.json similarity index 100% rename from pkg/models/validator/schemas/workflows/workflow-step.json rename to internal/playbooks/validator/schemas/workflows/workflow-step.json diff --git a/pkg/models/validator/validators_test.go b/internal/playbooks/validator/validators_test.go similarity index 99% rename from pkg/models/validator/validators_test.go rename to internal/playbooks/validator/validators_test.go index cd710758..ad7d9e40 100644 --- a/pkg/models/validator/validators_test.go +++ b/internal/playbooks/validator/validators_test.go @@ -6,7 +6,7 @@ import ( "fmt" "io" "os" - "soarca/pkg/models/cacao" + "soarca/pkg/cacao" "strings" "testing" diff --git a/internal/registry/registry_test.go b/internal/registry/registry_test.go new file mode 100644 index 00000000..ee3894fb --- /dev/null +++ b/internal/registry/registry_test.go @@ -0,0 +1,129 @@ +package registry + +import ( + "errors" + "testing" + + "github.com/go-playground/assert/v2" +) + +func TestRegisterAndGet(t *testing.T) { + r := New[string]() + + err := r.Register("exec1", "step1", "value1") + assert.Equal(t, err, nil) + + value, err := r.Get("exec1", "step1") + assert.Equal(t, err, nil) + assert.Equal(t, value, "value1") +} + +func TestRegisterDuplicateFails(t *testing.T) { + r := New[string]() + + err := r.Register("exec1", "step1", "value1") + assert.Equal(t, err, nil) + + err = r.Register("exec1", "step1", "value2") + assert.Equal(t, err, ErrAlreadyRegistered{OuterKey: "exec1", InnerKey: "step1"}) +} + +func TestRegisterSameOuterDifferentInner(t *testing.T) { + r := New[string]() + + assert.Equal(t, r.Register("exec1", "step1", "value1"), nil) + assert.Equal(t, r.Register("exec1", "step2", "value2"), nil) + + value1, err := r.Get("exec1", "step1") + assert.Equal(t, err, nil) + assert.Equal(t, value1, "value1") + + value2, err := r.Get("exec1", "step2") + assert.Equal(t, err, nil) + assert.Equal(t, value2, "value2") +} + +func TestGetMissingOuterKey(t *testing.T) { + r := New[string]() + + _, err := r.Get("missing", "step1") + assert.Equal(t, err, ErrOuterKeyNotFound{OuterKey: "missing"}) + + var target ErrOuterKeyNotFound + assert.Equal(t, errors.As(err, &target), true) +} + +func TestGetMissingInnerKey(t *testing.T) { + r := New[string]() + assert.Equal(t, r.Register("exec1", "step1", "value1"), nil) + + _, err := r.Get("exec1", "missing") + assert.Equal(t, err, ErrInnerKeyNotFound{OuterKey: "exec1", InnerKey: "missing"}) +} + +func TestRemove(t *testing.T) { + r := New[string]() + assert.Equal(t, r.Register("exec1", "step1", "value1"), nil) + + err := r.Remove("exec1", "step1") + assert.Equal(t, err, nil) + + _, err = r.Get("exec1", "step1") + assert.Equal(t, err, ErrOuterKeyNotFound{OuterKey: "exec1"}) +} + +func TestRemovePrunesOuterKeyOnlyWhenEmpty(t *testing.T) { + r := New[string]() + assert.Equal(t, r.Register("exec1", "step1", "value1"), nil) + assert.Equal(t, r.Register("exec1", "step2", "value2"), nil) + + err := r.Remove("exec1", "step1") + assert.Equal(t, err, nil) + + // exec1 should still exist because step2 remains registered. + value, err := r.Get("exec1", "step2") + assert.Equal(t, err, nil) + assert.Equal(t, value, "value2") + + err = r.Remove("exec1", "step2") + assert.Equal(t, err, nil) + + _, err = r.Get("exec1", "step2") + assert.Equal(t, err, ErrOuterKeyNotFound{OuterKey: "exec1"}) +} + +func TestRemoveMissingOuterKey(t *testing.T) { + r := New[string]() + + err := r.Remove("missing", "step1") + assert.Equal(t, err, ErrOuterKeyNotFound{OuterKey: "missing"}) +} + +func TestRemoveMissingInnerKey(t *testing.T) { + r := New[string]() + assert.Equal(t, r.Register("exec1", "step1", "value1"), nil) + + err := r.Remove("exec1", "missing") + assert.Equal(t, err, ErrInnerKeyNotFound{OuterKey: "exec1", InnerKey: "missing"}) +} + +func TestList(t *testing.T) { + r := New[string]() + + assert.Equal(t, len(r.List()), 0) + + assert.Equal(t, r.Register("exec1", "step1", "value1"), nil) + assert.Equal(t, r.Register("exec1", "step2", "value2"), nil) + assert.Equal(t, r.Register("exec2", "step1", "value3"), nil) + + values := r.List() + assert.Equal(t, len(values), 3) + + seen := map[string]bool{} + for _, v := range values { + seen[v] = true + } + assert.Equal(t, seen["value1"], true) + assert.Equal(t, seen["value2"], true) + assert.Equal(t, seen["value3"], true) +} diff --git a/internal/registry/store.go b/internal/registry/store.go new file mode 100644 index 00000000..21fccd68 --- /dev/null +++ b/internal/registry/store.go @@ -0,0 +1,138 @@ +// Package registry provides a small, generic, in-memory store for +// pending work items that are registered under a two-level key, looked +// up by callers that later resolve or expire them, and automatically +// pruned once their outer key has no remaining inner entries. +// +// It captures the shape used by SOARCA's manual-command manual inbox +// (keyed by run id -> step run id): register a pending item, +// let something external resolve it asynchronously (e.g. by pushing a +// result onto a channel stored alongside it), and clean up stale entries. +// +// The Fin job queue (pkg/core/capability/fin/queue) looks superficially +// similar - it is also a two-level, register/resolve/clean-up store keyed +// by capability type -> job id - but it is deliberately its own, +// independent implementation rather than an instantiation of Registry: it +// needs capability-type-filtered claiming by any number of competing +// pollers, per-job lease expiry with a background sweep-and-requeue loop, +// and broadcast wakeups for long-polling claimants, none of which fit this +// package's single-value-per-key, synchronous get/remove model. +package registry + +import ( + "fmt" + "sync" +) + +// ErrOuterKeyNotFound indicates no entries are registered under outerKey +// at all. +type ErrOuterKeyNotFound struct { + OuterKey string +} + +func (e ErrOuterKeyNotFound) Error() string { + return fmt.Sprintf("no entries found for key %q", e.OuterKey) +} + +// ErrInnerKeyNotFound indicates outerKey has registered entries, but none +// under innerKey. +type ErrInnerKeyNotFound struct { + OuterKey string + InnerKey string +} + +func (e ErrInnerKeyNotFound) Error() string { + return fmt.Sprintf("no entry found for key %q -> %q", e.OuterKey, e.InnerKey) +} + +// ErrAlreadyRegistered indicates an entry already exists under the given +// outerKey/innerKey pair. +type ErrAlreadyRegistered struct { + OuterKey string + InnerKey string +} + +func (e ErrAlreadyRegistered) Error() string { + return fmt.Sprintf("an entry is already registered for key %q -> %q", e.OuterKey, e.InnerKey) +} + +// Registry is a generic, two-level (outerKey -> innerKey -> value) +// in-memory store, safe for concurrent use. +type Registry[V any] struct { + mu sync.Mutex + entries map[string]map[string]V +} + +// New creates an empty Registry. +func New[V any]() *Registry[V] { + return &Registry[V]{entries: map[string]map[string]V{}} +} + +// Register adds value under outerKey/innerKey. It fails if an entry is +// already registered under that exact pair. +func (r *Registry[V]) Register(outerKey string, innerKey string, value V) error { + r.mu.Lock() + defer r.mu.Unlock() + + inner, ok := r.entries[outerKey] + if !ok { + r.entries[outerKey] = map[string]V{innerKey: value} + return nil + } + if _, exists := inner[innerKey]; exists { + return ErrAlreadyRegistered{OuterKey: outerKey, InnerKey: innerKey} + } + inner[innerKey] = value + return nil +} + +// Get retrieves the value registered under outerKey/innerKey. +func (r *Registry[V]) Get(outerKey string, innerKey string) (V, error) { + r.mu.Lock() + defer r.mu.Unlock() + + var zero V + inner, ok := r.entries[outerKey] + if !ok { + return zero, ErrOuterKeyNotFound{OuterKey: outerKey} + } + value, ok := inner[innerKey] + if !ok { + return zero, ErrInnerKeyNotFound{OuterKey: outerKey, InnerKey: innerKey} + } + return value, nil +} + +// Remove deletes the value registered under outerKey/innerKey. If this +// was the last remaining entry for outerKey, the outer key itself is +// pruned too, keeping the registry clean. +func (r *Registry[V]) Remove(outerKey string, innerKey string) error { + r.mu.Lock() + defer r.mu.Unlock() + + inner, ok := r.entries[outerKey] + if !ok { + return ErrOuterKeyNotFound{OuterKey: outerKey} + } + if _, ok := inner[innerKey]; !ok { + return ErrInnerKeyNotFound{OuterKey: outerKey, InnerKey: innerKey} + } + delete(inner, innerKey) + if len(inner) == 0 { + delete(r.entries, outerKey) + } + return nil +} + +// List returns all currently registered values, in no particular order. +func (r *Registry[V]) List() []V { + r.mu.Lock() + defer r.mu.Unlock() + + values := make([]V, 0) + for _, inner := range r.entries { + for _, value := range inner { + values = append(values, value) + } + } + return values +} diff --git a/pkg/reporting/cases/correlation/correlation_test.go b/internal/reporting/cases/correlation/correlation_test.go similarity index 83% rename from pkg/reporting/cases/correlation/correlation_test.go rename to internal/reporting/cases/correlation/correlation_test.go index 90988670..9996c8ee 100644 --- a/pkg/reporting/cases/correlation/correlation_test.go +++ b/internal/reporting/cases/correlation/correlation_test.go @@ -2,10 +2,10 @@ package correlation import ( "fmt" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" - "soarca/pkg/reporting/cases/incident" - "soarca/pkg/reporting/cases/observable" + "soarca/pkg/cacao" + "soarca/internal/runs/model" + "soarca/internal/reporting/cases/incident" + "soarca/internal/reporting/cases/observable" "soarca/test/unittest/mocks/mock_guid" "testing" @@ -18,7 +18,7 @@ func TestCorrelation(t *testing.T) { correlation := New(&guid) var1 := cacao.Variable{Name: "__SOURCE_IPV4__", Type: cacao.VariableTypeIpv4Address, Value: "10.0.0.1"} variables := cacao.NewVariables(var1) - meta := execution.Metadata{ExecutionId: uuid.MustParse("01983c7b-2983-7b7d-b3bb-548d2b5d7a2a"), + meta := run.Metadata{RunId: uuid.MustParse("01983c7b-2983-7b7d-b3bb-548d2b5d7a2a"), PlaybookId: "some-playbook", StepId: "SomeStep"} @@ -39,7 +39,7 @@ func TestAddSecondCorrelation(t *testing.T) { correlation := New(&guid) var1 := cacao.Variable{Name: "__SOURCE_IPV4__", Type: cacao.VariableTypeIpv4Address, Value: "10.0.0.1"} variables := cacao.NewVariables(var1) - meta := execution.Metadata{ExecutionId: uuid.MustParse("01983c7b-2983-7b7d-b3bb-548d2b5d7a2a"), + meta := run.Metadata{RunId: uuid.MustParse("01983c7b-2983-7b7d-b3bb-548d2b5d7a2a"), PlaybookId: "some-playbook", StepId: "SomeStep"} @@ -55,7 +55,7 @@ func TestAddSecondCorrelation(t *testing.T) { } var2 := cacao.Variable{Name: "__SOURCE_IPV4__", Type: cacao.VariableTypeIpv4Address, Value: "10.0.0.2"} - meta2 := execution.Metadata{ExecutionId: uuid.MustParse("01983caa-3cd6-704b-9598-ee9424166ead"), + meta2 := run.Metadata{RunId: uuid.MustParse("01983caa-3cd6-704b-9598-ee9424166ead"), PlaybookId: "some-playbook", StepId: "SomeStep"} variables2 := cacao.NewVariables(var2) @@ -73,7 +73,7 @@ func TestAddSecondCorrelation(t *testing.T) { ob1 := observable.Observable{Type: observable.SourceAddressIpv4, Name: var1.Name, Value: var1.Value, - Executions: observable.Executions{meta.ExecutionId: observable.Initial}} + Runs: observable.Runs{meta.RunId: observable.Initial}} obs1 := incident.Observables{} obs1[ob1.Value] = ob1 @@ -83,7 +83,7 @@ func TestAddSecondCorrelation(t *testing.T) { ob2 := observable.Observable{Type: observable.SourceAddressIpv4, Name: var2.Name, Value: var2.Value, - Executions: observable.Executions{meta2.ExecutionId: observable.Initial}} + Runs: observable.Runs{meta2.RunId: observable.Initial}} obs2 := incident.Observables{} obs2[ob2.Value] = ob2 @@ -96,7 +96,7 @@ func TestMultipleAdditionsToMultipleCases(t *testing.T) { correlation := New(&guid) var1 := cacao.Variable{Name: "__SOURCE_IPV4__", Type: cacao.VariableTypeIpv4Address, Value: "10.0.0.1"} variables := cacao.NewVariables(var1) - meta := execution.Metadata{ExecutionId: uuid.MustParse("01983c7b-2983-7b7d-b3bb-548d2b5d7a2a"), + meta := run.Metadata{RunId: uuid.MustParse("01983c7b-2983-7b7d-b3bb-548d2b5d7a2a"), PlaybookId: "some-playbook", StepId: "SomeStep"} @@ -112,7 +112,7 @@ func TestMultipleAdditionsToMultipleCases(t *testing.T) { } var2 := cacao.Variable{Name: "__SOURCE_IPV4__", Type: cacao.VariableTypeIpv4Address, Value: "10.0.0.2"} - meta2 := execution.Metadata{ExecutionId: uuid.MustParse("01983caa-3cd6-704b-9598-ee9424166ead"), + meta2 := run.Metadata{RunId: uuid.MustParse("01983caa-3cd6-704b-9598-ee9424166ead"), PlaybookId: "some-playbook", StepId: "SomeStep"} variables2 := cacao.NewVariables(var2) diff --git a/pkg/reporting/cases/correlation/correlation.go b/internal/reporting/cases/correlation/matcher.go similarity index 67% rename from pkg/reporting/cases/correlation/correlation.go rename to internal/reporting/cases/correlation/matcher.go index 2bec1977..f8c6dc61 100644 --- a/pkg/reporting/cases/correlation/correlation.go +++ b/internal/reporting/cases/correlation/matcher.go @@ -3,9 +3,9 @@ package correlation import ( "reflect" "soarca/internal/logger" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" - "soarca/pkg/reporting/cases/incident" + "soarca/pkg/cacao" + "soarca/internal/runs/model" + "soarca/internal/reporting/cases/incident" "soarca/pkg/utils/guid" @@ -22,7 +22,7 @@ func init() { } type ICorrelation interface { - Correlate(execution.Metadata, cacao.Variables) (string, error) + Correlate(run.Metadata, cacao.Variables) (string, error) } type ICorrelationIntegration interface { @@ -41,7 +41,7 @@ func New(guid guid.IGuid) Correlation { return Correlation{cases: Cases{}, guid: guid} } -func (correlation *Correlation) GetCaseId(meta execution.Metadata, playbook cacao.Playbook) string { +func (correlation *Correlation) GetCaseId(meta run.Metadata, playbook cacao.Playbook) string { internalId, err := correlation.Correlate(meta, playbook.PlaybookVariables) if err != nil { return "" @@ -52,14 +52,14 @@ func (correlation *Correlation) GetCaseId(meta execution.Metadata, playbook caca } -func (correlation *Correlation) Correlate(meta execution.Metadata, variables cacao.Variables) (string, error) { +func (correlation *Correlation) Correlate(meta run.Metadata, variables cacao.Variables) (string, error) { for _, item := range correlation.cases { for _, variable := range variables { if part, err := item.CheckObservableInCase(variable); err != nil { return "", err } else if part { log.Debug(part) - correlation.addExecutionToCase(item, meta, variable) + correlation.addRunToCase(item, meta, variable) return item.GetId().String(), nil } } @@ -69,7 +69,7 @@ func (correlation *Correlation) Correlate(meta execution.Metadata, variables cac return id.String(), nil } -// func (correlation *Correlation) addEddxecutionToCase(meta execution.Metadata, variables cacao.Variables) error { +// func (correlation *Correlation) addEddxecutionToCase(meta run.Metadata, variables cacao.Variables) error { // return nil // } @@ -77,7 +77,7 @@ func (correlation *Correlation) Correlate(meta execution.Metadata, variables cac // } -func (correlation *Correlation) addExecutionToCase(item incident.ICase, meta execution.Metadata, variable cacao.Variable) { +func (correlation *Correlation) addRunToCase(item incident.ICase, meta run.Metadata, variable cacao.Variable) { added, err := correlation.cases[item.GetId()].AddIfNotInCase(meta, variable) if err != nil { log.Error(err) @@ -85,7 +85,7 @@ func (correlation *Correlation) addExecutionToCase(item incident.ICase, meta exe log.Info(added) } -func (correlation *Correlation) createCase(meta execution.Metadata, variables cacao.Variables) uuid.UUID { +func (correlation *Correlation) createCase(meta run.Metadata, variables cacao.Variables) uuid.UUID { thisCase := incident.New(correlation.guid.NewV7(), meta, variables) correlation.cases[thisCase.GetId()] = thisCase return thisCase.GetId() diff --git a/pkg/reporting/cases/incident/incident_test.go b/internal/reporting/cases/incident/incident_test.go similarity index 81% rename from pkg/reporting/cases/incident/incident_test.go rename to internal/reporting/cases/incident/incident_test.go index aa50c116..052639f4 100644 --- a/pkg/reporting/cases/incident/incident_test.go +++ b/internal/reporting/cases/incident/incident_test.go @@ -2,9 +2,9 @@ package incident import ( "fmt" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" - "soarca/pkg/reporting/cases/observable" + "soarca/pkg/cacao" + "soarca/internal/runs/model" + "soarca/internal/reporting/cases/observable" "testing" "time" @@ -17,7 +17,7 @@ func TestCreateCase(t *testing.T) { caseId, _ := uuid.NewV7() var1 := cacao.Variable{Name: "__SOURCE_IPV4__", Type: cacao.VariableTypeIpv4Address, Value: "10.0.0.1"} variables := cacao.NewVariables(var1) - meta := execution.Metadata{ExecutionId: uuid.MustParse("01983c5a-067f-7016-a29b-10801077e6d6"), + meta := run.Metadata{RunId: uuid.MustParse("01983c5a-067f-7016-a29b-10801077e6d6"), PlaybookId: "some-playbook", StepId: "SomeStep"} thisCase := New(caseId, meta, variables) @@ -48,14 +48,14 @@ func TestIfObservableIsInCase(t *testing.T) { func TestDouble(t *testing.T) { - meta1 := execution.Metadata{ExecutionId: uuid.MustParse("01983c5a-067f-7016-a29b-10801077e6d6"), + meta1 := run.Metadata{RunId: uuid.MustParse("01983c5a-067f-7016-a29b-10801077e6d6"), PlaybookId: "some-playbook", StepId: "SomeStep"} ip1 := observable.Observable{Type: observable.SourceAddressIpv4, Name: "__DESTINATION_IPV4__", Value: "10.0.0.1", - Executions: observable.Executions{meta1.ExecutionId: observable.Initial}} + Runs: observable.Runs{meta1.RunId: observable.Initial}} observables := make(map[string]observable.Observable) observables[ip1.Value] = ip1 @@ -68,7 +68,7 @@ func TestDouble(t *testing.T) { assert.Equal(t, err, nil) assert.Equal(t, result, true) - meta2 := execution.Metadata{ExecutionId: uuid.MustParse("01983c5a-067f-7016-a29b-10801077e6d6"), + meta2 := run.Metadata{RunId: uuid.MustParse("01983c5a-067f-7016-a29b-10801077e6d6"), PlaybookId: "some-playbook", StepId: "SomeStep"} diff --git a/pkg/reporting/cases/incident/incident.go b/internal/reporting/cases/incident/record.go similarity index 79% rename from pkg/reporting/cases/incident/incident.go rename to internal/reporting/cases/incident/record.go index 2387311c..5ab05c46 100644 --- a/pkg/reporting/cases/incident/incident.go +++ b/internal/reporting/cases/incident/record.go @@ -4,9 +4,9 @@ import ( "errors" "reflect" "soarca/internal/logger" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" - "soarca/pkg/reporting/cases/observable" + "soarca/pkg/cacao" + "soarca/internal/runs/model" + "soarca/internal/reporting/cases/observable" "time" @@ -24,7 +24,7 @@ func init() { type ICase interface { CheckObservableInCase(cacao.Variable) (bool, error) - AddIfNotInCase(execution.Metadata, cacao.Variable) (bool, error) + AddIfNotInCase(run.Metadata, cacao.Variable) (bool, error) GetId() uuid.UUID GetExternalId() string GetObservables() Observables @@ -37,11 +37,11 @@ type StepResult struct { } type Result struct { - Meta execution.Metadata + Meta run.Metadata StepResults map[string]StepResult } -type Executions map[uuid.UUID]execution.Metadata +type Runs map[uuid.UUID]run.Metadata type Observables map[string]observable.Observable type Case struct { @@ -49,13 +49,13 @@ type Case struct { ExternalId string Observables map[string]observable.Observable FirstObserved time.Time - Executions Executions + Runs Runs // time itime.ITime // IsClosed bool } func New(guid uuid.UUID, - meta execution.Metadata, + meta run.Metadata, variables cacao.Variables) ICase { observables := Observables{} thisCase := Case{Id: guid, @@ -88,14 +88,14 @@ func (cas *Case) CheckObservableInCase(variable cacao.Variable) (bool, error) { return false, nil } -func (cas *Case) AddIfNotInCase(meta execution.Metadata, +func (cas *Case) AddIfNotInCase(meta run.Metadata, variable cacao.Variable) (bool, error) { if _, ok := cas.Observables[variable.Value]; !ok { observed, err := observable.CreateObservable(variable) if err != nil { return false, err } - observed.AddExecutionToObservable(meta.ExecutionId, observable.Initial) + observed.AddRunToObservable(meta.RunId, observable.Initial) cas.Observables[variable.Value] = observed return true, nil @@ -103,7 +103,7 @@ func (cas *Case) AddIfNotInCase(meta execution.Metadata, return false, nil } -func (cas *Case) AddExecutionAndStepData(variable cacao.Variable, step cacao.Step, meta execution.Metadata) { +func (cas *Case) AddRunAndStepData(variable cacao.Variable, step cacao.Step, meta run.Metadata) { newObserved, err := observable.CreateObservable(variable) if err != nil { log.Error(err) @@ -111,13 +111,13 @@ func (cas *Case) AddExecutionAndStepData(variable cacao.Variable, step cacao.Ste } if observed, ok := cas.Observables[variable.Value]; ok { - err := observed.Match(newObserved, meta.ExecutionId) + err := observed.Match(newObserved, meta.RunId) if err != nil { log.Error(err) return } cas.Observables[variable.Value] = observed - cas.Executions[meta.ExecutionId] = meta + cas.Runs[meta.RunId] = meta } } diff --git a/pkg/reporting/cases/cases.go b/internal/reporting/cases/manager.go similarity index 73% rename from pkg/reporting/cases/cases.go rename to internal/reporting/cases/manager.go index 88a89f46..510b0353 100644 --- a/pkg/reporting/cases/cases.go +++ b/internal/reporting/cases/manager.go @@ -1,9 +1,9 @@ package cases import ( - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" - "soarca/pkg/reporting/cases/observable" + "soarca/pkg/cacao" + "soarca/internal/runs/model" + "soarca/internal/reporting/cases/observable" "github.com/gofrs/uuid" ) @@ -13,7 +13,7 @@ const SOARCA_PLAYBOOK_CASE_ID = "__SOARCA_CASE_ID__" type Cases map[uuid.UUID]ICase type ICasesManager interface { - AddToExistingOrCreateNew(execution.Metadata, cacao.Playbook) cacao.Variable + AddToExistingOrCreateNew(run.Metadata, cacao.Playbook) cacao.Variable } type CaseIds struct { @@ -33,7 +33,7 @@ func (caseIds *CaseIds) Remove(backendName string) { type ICase interface { CheckObservableInCase(cacao.Variable) (bool, error) - AddIfNotInCase(execution.Metadata, cacao.Variable) (bool, error) + AddIfNotInCase(run.Metadata, cacao.Variable) (bool, error) GetId() uuid.UUID GetExternalId() string } diff --git a/pkg/reporting/cases/observable/observable_test.go b/internal/reporting/cases/observable/observable_test.go similarity index 66% rename from pkg/reporting/cases/observable/observable_test.go rename to internal/reporting/cases/observable/observable_test.go index d0a97b53..1585bc97 100644 --- a/pkg/reporting/cases/observable/observable_test.go +++ b/internal/reporting/cases/observable/observable_test.go @@ -1,25 +1,25 @@ package observable import ( - "soarca/pkg/models/cacao" + "soarca/pkg/cacao" "testing" "github.com/go-playground/assert/v2" "github.com/google/uuid" ) -func TestExecutions(t *testing.T) { +func TestRuns(t *testing.T) { id, err := uuid.NewV7() assert.Equal(t, err, nil) - obs := Observable{Type: SourceAddressIpv4, Name: "__SRC_IP__", Value: "10.0.0.1", Executions: Executions{id: Initial}} + obs := Observable{Type: SourceAddressIpv4, Name: "__SRC_IP__", Value: "10.0.0.1", Runs: Runs{id: Initial}} id2, err := uuid.NewV7() assert.Equal(t, err, nil) - obs.AddExecutionToObservable(id2, SameObservable) + obs.AddRunToObservable(id2, SameObservable) - assert.Equal(t, obs.Executions[id], Initial) - assert.Equal(t, obs.Executions[id2], SameObservable) + assert.Equal(t, obs.Runs[id], Initial) + assert.Equal(t, obs.Runs[id2], SameObservable) } @@ -32,19 +32,19 @@ func TestCreate(t *testing.T) { id2, err := uuid.NewV7() assert.Equal(t, err, nil) - obs.AddExecutionToObservable(id2, SameObservable) - assert.Equal(t, obs.Executions[id2], SameObservable) + obs.AddRunToObservable(id2, SameObservable) + assert.Equal(t, obs.Runs[id2], SameObservable) } func TestMatchSame(t *testing.T) { id, err := uuid.NewV7() assert.Equal(t, err, nil) - obs := Observable{Type: SourceAddressIpv4, Name: "__SRC_IP__", Value: "10.0.0.1", Executions: Executions{id: Initial}} + obs := Observable{Type: SourceAddressIpv4, Name: "__SRC_IP__", Value: "10.0.0.1", Runs: Runs{id: Initial}} id2, err := uuid.NewV7() assert.Equal(t, err, nil) - obs2 := Observable{Type: SourceAddressIpv4, Name: "__SRC_IP__", Value: "10.0.0.1", Executions: Executions{id2: Initial}} + obs2 := Observable{Type: SourceAddressIpv4, Name: "__SRC_IP__", Value: "10.0.0.1", Runs: Runs{id2: Initial}} err = obs.Match(obs2, id2) assert.Equal(t, err, nil) @@ -54,11 +54,11 @@ func TestMatchSame(t *testing.T) { func TestMatchLateral(t *testing.T) { id, err := uuid.NewV7() assert.Equal(t, err, nil) - obs := Observable{Type: DestinationAddressIpv4, Name: "__DEST_IP__", Value: "10.0.0.1", Executions: Executions{id: Initial}} + obs := Observable{Type: DestinationAddressIpv4, Name: "__DEST_IP__", Value: "10.0.0.1", Runs: Runs{id: Initial}} id2, err := uuid.NewV7() assert.Equal(t, err, nil) - obs2 := Observable{Type: SourceAddressIpv4, Name: "__SRC_IP__", Value: "10.0.0.1", Executions: Executions{id2: Initial}} + obs2 := Observable{Type: SourceAddressIpv4, Name: "__SRC_IP__", Value: "10.0.0.1", Runs: Runs{id2: Initial}} err = obs.Match(obs2, id2) assert.Equal(t, err, nil) } diff --git a/pkg/reporting/cases/observable/observable.go b/internal/reporting/cases/observable/record.go similarity index 86% rename from pkg/reporting/cases/observable/observable.go rename to internal/reporting/cases/observable/record.go index 2b5a01a3..a47253c9 100644 --- a/pkg/reporting/cases/observable/observable.go +++ b/internal/reporting/cases/observable/record.go @@ -5,7 +5,7 @@ import ( "reflect" "slices" "soarca/internal/logger" - "soarca/pkg/models/cacao" + "soarca/pkg/cacao" "github.com/google/uuid" ) @@ -21,7 +21,7 @@ func init() { type ObservableType int type MatchReason string -type Executions map[uuid.UUID]MatchReason +type Runs map[uuid.UUID]MatchReason var ip_v4_source_type_ids = []string{"__SOURCE_IPV4__", "__SRC_IPV4__"} var ip_v4_destination_ids = []string{"__DESTINATION_IPV4__", "__DEST_IPV4__"} @@ -51,7 +51,7 @@ type Observable struct { Type ObservableType Name string Value string - Executions Executions + Runs Runs } func deductObservableType(name string) ObservableType { @@ -82,27 +82,27 @@ func CreateObservable(variable cacao.Variable) (Observable, error) { return Observable{Type: deductObservableType(variable.Name), Name: variable.Name, Value: variable.Value, - Executions: Executions{}}, nil + Runs: Runs{}}, nil } -func (observable *Observable) AddExecutionToObservable(id uuid.UUID, match MatchReason) { - if _, ok := observable.Executions[id]; ok { +func (observable *Observable) AddRunToObservable(id uuid.UUID, match MatchReason) { + if _, ok := observable.Runs[id]; ok { return } - observable.Executions[id] = match + observable.Runs[id] = match } func (observable *Observable) Match(other Observable, exectuion uuid.UUID) error { - if len(other.Executions) > 0 { - log.Warning("the other executions will be discarded when updating the base") + if len(other.Runs) > 0 { + log.Warning("the other runs will be discarded when updating the base") } reason := determineMatchReason(*observable, other) if reason == NoMatch { return errors.New("no match made with observables") } - observable.Executions[exectuion] = reason + observable.Runs[exectuion] = reason log.Debug("Matched observable: ", observable) return nil } diff --git a/pkg/reporting/reporter/downstream_reporter/cache/cache_test.go b/internal/reporting/reporter/downstream_reporter/runstate/cache_test.go similarity index 71% rename from pkg/reporting/reporter/downstream_reporter/cache/cache_test.go rename to internal/reporting/reporter/downstream_reporter/runstate/cache_test.go index e71b340d..31e32da8 100644 --- a/pkg/reporting/reporter/downstream_reporter/cache/cache_test.go +++ b/internal/reporting/reporter/downstream_reporter/runstate/cache_test.go @@ -1,10 +1,11 @@ -package cache +package runstate import ( b64 "encoding/base64" "errors" - "soarca/pkg/models/cacao" - cache_model "soarca/pkg/models/cache" + "soarca/pkg/cacao" + runstate_model "soarca/internal/runs/state" + "soarca/internal/runs/model" mock_time "soarca/test/unittest/mocks/mock_utils/time" "testing" "time" @@ -16,7 +17,7 @@ import ( func TestReportWorkflowStartFirst(t *testing.T) { mock_time := new(mock_time.MockTime) - cacheReporter := New(mock_time, 10) + runStateReporter := New(mock_time, 10) expectedCommand := cacao.Command{ Type: "ssh", @@ -76,36 +77,39 @@ func TestReportWorkflowStartFirst(t *testing.T) { Workflow: map[string]cacao.Step{step1.ID: step1, end.ID: end}, } - executionId0 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c0") + runId0 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c0") + stepRunId0 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c9") + metadata0 := run.Metadata{RunId: runId0, StepId: step1.ID, StepRunId: stepRunId0} layout := "2006-01-02T15:04:05.000Z" str := "2014-11-12T11:45:26.371Z" timeNow, _ := time.Parse(layout, str) mock_time.On("Now").Return(timeNow) - err := cacheReporter.ReportWorkflowStart(executionId0, playbook, mock_time.Now()) + err := runStateReporter.ReportWorkflowStart(runId0, playbook, mock_time.Now()) if err != nil { t.Fail() } mock_time.On("Now").Return(timeNow) - err = cacheReporter.ReportStepStart(executionId0, step1, cacao.NewVariables(expectedVariables), mock_time.Now()) + err = runStateReporter.ReportStepStart(metadata0, step1, cacao.NewVariables(expectedVariables), mock_time.Now()) if err != nil { t.Fail() } mock_time.On("Now").Return(timeNow) - err = cacheReporter.ReportStepEnd(executionId0, step1, cacao.NewVariables(), nil, mock_time.Now()) + err = runStateReporter.ReportStepEnd(metadata0, step1, cacao.NewVariables(), nil, mock_time.Now()) if err != nil { t.Fail() } expectedStarted, _ := time.Parse(layout, "2014-11-12T11:45:26.371Z") expectedEnded, _ := time.Parse(layout, "0001-01-01T00:00:00Z") - expetedStepReport := cache_model.StepResult{ - ExecutionId: executionId0, + expetedStepReport := runstate_model.StepResult{ + RunId: runId0, StepId: "action--test", + StepRunId: stepRunId0, Name: "ssh-tests", Description: "test step", IsAutomated: true, @@ -113,42 +117,42 @@ func TestReportWorkflowStartFirst(t *testing.T) { Ended: timeNow, CommandsB64: []string{b64.StdEncoding.EncodeToString([]byte(expectedCommand.Command))}, Variables: cacao.NewVariables(), - Status: cache_model.SuccessfullyExecuted, + Status: runstate_model.SuccessfullyExecuted, Error: nil, } - expectedExecutions := []cache_model.ExecutionEntry{ + expectedRuns := []runstate_model.RunEntry{ { - ExecutionId: executionId0, + RunId: runId0, PlaybookId: "test", Name: "ssh-test-playbook", Description: "Playbook description", Started: expectedStarted, Ended: expectedEnded, - StepResults: map[string]cache_model.StepResult{expetedStepReport.StepId: expetedStepReport}, + StepResults: map[string]runstate_model.StepResult{expetedStepReport.StepRunId.String(): expetedStepReport}, Error: nil, Status: 2, }, } - returnedExecutions, _ := cacheReporter.GetExecutions() + returnedRuns, _ := runStateReporter.GetRuns() - exec, err := cacheReporter.GetExecutionReport(executionId0) - assert.Equal(t, expectedExecutions, returnedExecutions) - assert.Equal(t, len(expectedExecutions), 1) - assert.Equal(t, expectedExecutions[0].ExecutionId, exec.ExecutionId) - assert.Equal(t, expectedExecutions[0].PlaybookId, exec.PlaybookId) - assert.Equal(t, expectedExecutions[0].StepResults, exec.StepResults) - assert.Equal(t, expectedExecutions[0].Started, timeNow) - assert.Equal(t, expectedExecutions[0].Ended, time.Time{}) - assert.Equal(t, expectedExecutions[0].Status, exec.Status) + exec, err := runStateReporter.GetRunReport(runId0) + assert.Equal(t, expectedRuns, returnedRuns) + assert.Equal(t, len(expectedRuns), 1) + assert.Equal(t, expectedRuns[0].RunId, exec.RunId) + assert.Equal(t, expectedRuns[0].PlaybookId, exec.PlaybookId) + assert.Equal(t, expectedRuns[0].StepResults, exec.StepResults) + assert.Equal(t, expectedRuns[0].Started, timeNow) + assert.Equal(t, expectedRuns[0].Ended, time.Time{}) + assert.Equal(t, expectedRuns[0].Status, exec.Status) assert.Equal(t, err, nil) mock_time.AssertExpectations(t) } func TestReportWorkflowStartFifo(t *testing.T) { mock_time := new(mock_time.MockTime) - cacheReporter := New(mock_time, 3) + runStateReporter := New(mock_time, 3) expectedCommand := cacao.Command{ Type: "ssh", @@ -208,93 +212,93 @@ func TestReportWorkflowStartFifo(t *testing.T) { Workflow: map[string]cacao.Step{step1.ID: step1, end.ID: end}, } - executionId0 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c0") - executionId1 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c1") - executionId2 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c2") - executionId3 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c3") + runId0 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c0") + runId1 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c1") + runId2 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c2") + runId3 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c3") layout := "2006-01-02T15:04:05.000Z" str := "2014-11-12T11:45:26.371Z" timeNow, _ := time.Parse(layout, str) mock_time.On("Now").Return(timeNow) - executionIds := []uuid.UUID{ - executionId0, - executionId1, - executionId2, - executionId3, + runIds := []uuid.UUID{ + runId0, + runId1, + runId2, + runId3, } expectedStarted, _ := time.Parse(layout, "2014-11-12T11:45:26.371Z") expectedEnded, _ := time.Parse(layout, "0001-01-01T00:00:00Z") - expectedExecutionsFull := []cache_model.ExecutionEntry{} - for _, executionId := range executionIds[:len(executionIds)-1] { - t.Log(executionId) - entry := cache_model.ExecutionEntry{ - ExecutionId: executionId, + expectedRunsFull := []runstate_model.RunEntry{} + for _, runId := range runIds[:len(runIds)-1] { + t.Log(runId) + entry := runstate_model.RunEntry{ + RunId: runId, PlaybookId: "test", Name: "ssh-test-playbook", Description: "Playbook description", Started: expectedStarted, Ended: expectedEnded, - StepResults: map[string]cache_model.StepResult{}, + StepResults: map[string]runstate_model.StepResult{}, Error: nil, Status: 2, } - expectedExecutionsFull = append(expectedExecutionsFull, entry) + expectedRunsFull = append(expectedRunsFull, entry) } t.Log("") - expectedExecutionsFifo := []cache_model.ExecutionEntry{} - for _, executionId := range executionIds[1:] { - t.Log(executionId) - entry := cache_model.ExecutionEntry{ - ExecutionId: executionId, + expectedRunsFifo := []runstate_model.RunEntry{} + for _, runId := range runIds[1:] { + t.Log(runId) + entry := runstate_model.RunEntry{ + RunId: runId, PlaybookId: "test", Name: "ssh-test-playbook", Description: "Playbook description", Started: expectedStarted, Ended: expectedEnded, - StepResults: map[string]cache_model.StepResult{}, + StepResults: map[string]runstate_model.StepResult{}, Error: nil, Status: 2, } - expectedExecutionsFifo = append(expectedExecutionsFifo, entry) + expectedRunsFifo = append(expectedRunsFifo, entry) } - err := cacheReporter.ReportWorkflowStart(executionId0, playbook, mock_time.Now()) + err := runStateReporter.ReportWorkflowStart(runId0, playbook, mock_time.Now()) if err != nil { t.Fail() } - err = cacheReporter.ReportWorkflowStart(executionId1, playbook, mock_time.Now()) + err = runStateReporter.ReportWorkflowStart(runId1, playbook, mock_time.Now()) if err != nil { t.Fail() } - err = cacheReporter.ReportWorkflowStart(executionId2, playbook, mock_time.Now()) + err = runStateReporter.ReportWorkflowStart(runId2, playbook, mock_time.Now()) if err != nil { t.Fail() } - returnedExecutionsFull, _ := cacheReporter.GetExecutions() + returnedRunsFull, _ := runStateReporter.GetRuns() t.Log("expected") - t.Log(expectedExecutionsFull) + t.Log(expectedRunsFull) t.Log("returned") - t.Log(returnedExecutionsFull) - assert.Equal(t, expectedExecutionsFull, returnedExecutionsFull) + t.Log(returnedRunsFull) + assert.Equal(t, expectedRunsFull, returnedRunsFull) - err = cacheReporter.ReportWorkflowStart(executionId3, playbook, mock_time.Now()) + err = runStateReporter.ReportWorkflowStart(runId3, playbook, mock_time.Now()) if err != nil { t.Fail() } - returnedExecutionsFifo, _ := cacheReporter.GetExecutions() - assert.Equal(t, expectedExecutionsFifo, returnedExecutionsFifo) + returnedRunsFifo, _ := runStateReporter.GetRuns() + assert.Equal(t, expectedRunsFifo, returnedRunsFifo) mock_time.AssertExpectations(t) } func TestReportWorkflowEnd(t *testing.T) { mock_time := new(mock_time.MockTime) - cacheReporter := New(mock_time, 10) + runStateReporter := New(mock_time, 10) expectedCommand := cacao.Command{ Type: "ssh", @@ -354,50 +358,50 @@ func TestReportWorkflowEnd(t *testing.T) { Workflow: map[string]cacao.Step{step1.ID: step1, end.ID: end}, } - executionId0 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c0") + runId0 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c0") layout := "2006-01-02T15:04:05.000Z" str := "2014-11-12T11:45:26.371Z" timeNow, _ := time.Parse(layout, str) mock_time.On("Now").Return(timeNow) - err := cacheReporter.ReportWorkflowStart(executionId0, playbook, mock_time.Now()) + err := runStateReporter.ReportWorkflowStart(runId0, playbook, mock_time.Now()) if err != nil { t.Fail() } - err = cacheReporter.ReportWorkflowEnd(executionId0, playbook, nil, mock_time.Now()) + err = runStateReporter.ReportWorkflowEnd(runId0, playbook, nil, mock_time.Now()) if err != nil { t.Fail() } - expectedExecutionEntry := cache_model.ExecutionEntry{ - ExecutionId: executionId0, + expectedRunEntry := runstate_model.RunEntry{ + RunId: runId0, PlaybookId: "test", Name: "ssh-test-playbook", Description: "Playbook description", Started: timeNow, Ended: timeNow, - StepResults: map[string]cache_model.StepResult{}, - Status: cache_model.SuccessfullyExecuted, + StepResults: map[string]runstate_model.StepResult{}, + Status: runstate_model.SuccessfullyExecuted, } - expectedExecutions := []cache_model.ExecutionEntry{expectedExecutionEntry} + expectedRuns := []runstate_model.RunEntry{expectedRunEntry} - returnedExecutions, _ := cacheReporter.GetExecutions() + returnedRuns, _ := runStateReporter.GetRuns() - exec, err := cacheReporter.GetExecutionReport(executionId0) - assert.Equal(t, expectedExecutions, returnedExecutions) - assert.Equal(t, expectedExecutionEntry.ExecutionId, exec.ExecutionId) - assert.Equal(t, expectedExecutionEntry.PlaybookId, exec.PlaybookId) - assert.Equal(t, expectedExecutionEntry.StepResults, exec.StepResults) - assert.Equal(t, expectedExecutionEntry.Status, exec.Status) - assert.Equal(t, exec.Ended, expectedExecutionEntry.Ended) + exec, err := runStateReporter.GetRunReport(runId0) + assert.Equal(t, expectedRuns, returnedRuns) + assert.Equal(t, expectedRunEntry.RunId, exec.RunId) + assert.Equal(t, expectedRunEntry.PlaybookId, exec.PlaybookId) + assert.Equal(t, expectedRunEntry.StepResults, exec.StepResults) + assert.Equal(t, expectedRunEntry.Status, exec.Status) + assert.Equal(t, exec.Ended, expectedRunEntry.Ended) assert.Equal(t, err, nil) mock_time.AssertExpectations(t) } func TestReportStepStartAndEnd(t *testing.T) { mock_time := new(mock_time.MockTime) - cacheReporter := New(mock_time, 10) + runStateReporter := New(mock_time, 10) expectedCommand := cacao.Command{ Type: "ssh", @@ -455,34 +459,37 @@ func TestReportStepStartAndEnd(t *testing.T) { Workflow: map[string]cacao.Step{step1.ID: step1, end.ID: end}, } - executionId0 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c0") + runId0 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c0") + stepRunId0 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c9") + metadata0 := run.Metadata{RunId: runId0, StepId: step1.ID, StepRunId: stepRunId0} layout := "2006-01-02T15:04:05.000Z" str := "2014-11-12T11:45:26.371Z" timeNow, _ := time.Parse(layout, str) mock_time.On("Now").Return(timeNow) - err := cacheReporter.ReportWorkflowStart(executionId0, playbook, mock_time.Now()) + err := runStateReporter.ReportWorkflowStart(runId0, playbook, mock_time.Now()) if err != nil { t.Fail() } - err = cacheReporter.ReportStepStart(executionId0, step1, cacao.NewVariables(expectedVariables), mock_time.Now()) + err = runStateReporter.ReportStepStart(metadata0, step1, cacao.NewVariables(expectedVariables), mock_time.Now()) if err != nil { t.Fail() } - expectedStepStatus := cache_model.StepResult{ - ExecutionId: executionId0, - StepId: step1.ID, - Started: timeNow, - Ended: time.Time{}, - Variables: cacao.NewVariables(expectedVariables), - Status: cache_model.Ongoing, - Error: nil, + expectedStepStatus := runstate_model.StepResult{ + RunId: runId0, + StepId: step1.ID, + StepRunId: stepRunId0, + Started: timeNow, + Ended: time.Time{}, + Variables: cacao.NewVariables(expectedVariables), + Status: runstate_model.Ongoing, + Error: nil, } - exec, err := cacheReporter.GetExecutionReport(executionId0) - stepStatus := exec.StepResults[step1.ID] - assert.Equal(t, stepStatus.ExecutionId, expectedStepStatus.ExecutionId) + exec, err := runStateReporter.GetRunReport(runId0) + stepStatus := exec.StepResults[stepRunId0.String()] + assert.Equal(t, stepStatus.RunId, expectedStepStatus.RunId) assert.Equal(t, stepStatus.StepId, expectedStepStatus.StepId) assert.Equal(t, stepStatus.Started, expectedStepStatus.Started) assert.Equal(t, stepStatus.Ended, expectedStepStatus.Ended) @@ -491,24 +498,25 @@ func TestReportStepStartAndEnd(t *testing.T) { assert.Equal(t, stepStatus.Error, expectedStepStatus.Error) assert.Equal(t, err, nil) - err = cacheReporter.ReportStepEnd(executionId0, step1, cacao.NewVariables(expectedVariables), nil, mock_time.Now()) + err = runStateReporter.ReportStepEnd(metadata0, step1, cacao.NewVariables(expectedVariables), nil, mock_time.Now()) if err != nil { t.Fail() } - expectedStepResult := cache_model.StepResult{ - ExecutionId: executionId0, - StepId: step1.ID, - Started: timeNow, - Ended: timeNow, - Variables: cacao.NewVariables(expectedVariables), - Status: cache_model.SuccessfullyExecuted, - Error: nil, + expectedStepResult := runstate_model.StepResult{ + RunId: runId0, + StepId: step1.ID, + StepRunId: stepRunId0, + Started: timeNow, + Ended: timeNow, + Variables: cacao.NewVariables(expectedVariables), + Status: runstate_model.SuccessfullyExecuted, + Error: nil, } - exec, err = cacheReporter.GetExecutionReport(executionId0) - stepResult := exec.StepResults[step1.ID] - assert.Equal(t, stepResult.ExecutionId, expectedStepResult.ExecutionId) + exec, err = runStateReporter.GetRunReport(runId0) + stepResult := exec.StepResults[stepRunId0.String()] + assert.Equal(t, stepResult.RunId, expectedStepResult.RunId) assert.Equal(t, stepResult.StepId, expectedStepResult.StepId) assert.Equal(t, stepResult.Started, expectedStepResult.Started) assert.Equal(t, stepResult.Ended, expectedStepResult.Ended) @@ -521,7 +529,7 @@ func TestReportStepStartAndEnd(t *testing.T) { func TestReportStepStartCommandsEncoding(t *testing.T) { mock_time := new(mock_time.MockTime) - cacheReporter := New(mock_time, 10) + runStateReporter := New(mock_time, 10) expectedCommand1 := cacao.Command{ Type: "manual", @@ -583,17 +591,19 @@ func TestReportStepStartCommandsEncoding(t *testing.T) { Workflow: map[string]cacao.Step{step1.ID: step1, end.ID: end}, } - executionId0 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c0") + runId0 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c0") + stepRunId0 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c9") + metadata0 := run.Metadata{RunId: runId0, StepId: step1.ID, StepRunId: stepRunId0} layout := "2006-01-02T15:04:05.000Z" str := "2014-11-12T11:45:26.371Z" timeNow, _ := time.Parse(layout, str) mock_time.On("Now").Return(timeNow) - err := cacheReporter.ReportWorkflowStart(executionId0, playbook, mock_time.Now()) + err := runStateReporter.ReportWorkflowStart(runId0, playbook, mock_time.Now()) if err != nil { t.Fail() } - err = cacheReporter.ReportStepStart(executionId0, step1, cacao.NewVariables(expectedVariables), mock_time.Now()) + err = runStateReporter.ReportStepStart(metadata0, step1, cacao.NewVariables(expectedVariables), mock_time.Now()) if err != nil { t.Fail() } @@ -602,25 +612,26 @@ func TestReportStepStartCommandsEncoding(t *testing.T) { encodedCommand2 := b64.StdEncoding.EncodeToString([]byte(expectedCommand2.Command)) expectedCommandsB64 := []string{encodedCommand1, encodedCommand2} - expectedStepStatus := cache_model.StepResult{ - ExecutionId: executionId0, + expectedStepStatus := runstate_model.StepResult{ + RunId: runId0, StepId: step1.ID, + StepRunId: stepRunId0, Started: timeNow, Ended: time.Time{}, Variables: cacao.NewVariables(expectedVariables), - Status: cache_model.Ongoing, + Status: runstate_model.Ongoing, CommandsB64: expectedCommandsB64, Error: nil, IsAutomated: false, } - exec, err := cacheReporter.GetExecutionReport(executionId0) - stepStatus := exec.StepResults[step1.ID] + exec, err := runStateReporter.GetRunReport(runId0) + stepStatus := exec.StepResults[stepRunId0.String()] t.Log("stepStatus commands") t.Log(stepStatus.CommandsB64) t.Log("expectedStep commands") t.Log(expectedStepStatus.CommandsB64) - assert.Equal(t, stepStatus.ExecutionId, expectedStepStatus.ExecutionId) + assert.Equal(t, stepStatus.RunId, expectedStepStatus.RunId) assert.Equal(t, stepStatus.StepId, expectedStepStatus.StepId) assert.Equal(t, stepStatus.Started, expectedStepStatus.Started) assert.Equal(t, stepStatus.Ended, expectedStepStatus.Ended) @@ -631,24 +642,25 @@ func TestReportStepStartCommandsEncoding(t *testing.T) { assert.Equal(t, stepStatus.IsAutomated, expectedStepStatus.IsAutomated) assert.Equal(t, err, nil) - err = cacheReporter.ReportStepEnd(executionId0, step1, cacao.NewVariables(expectedVariables), nil, mock_time.Now()) + err = runStateReporter.ReportStepEnd(metadata0, step1, cacao.NewVariables(expectedVariables), nil, mock_time.Now()) if err != nil { t.Fail() } - expectedStepResult := cache_model.StepResult{ - ExecutionId: executionId0, - StepId: step1.ID, - Started: timeNow, - Ended: timeNow, - Variables: cacao.NewVariables(expectedVariables), - Status: cache_model.SuccessfullyExecuted, - Error: nil, + expectedStepResult := runstate_model.StepResult{ + RunId: runId0, + StepId: step1.ID, + StepRunId: stepRunId0, + Started: timeNow, + Ended: timeNow, + Variables: cacao.NewVariables(expectedVariables), + Status: runstate_model.SuccessfullyExecuted, + Error: nil, } - exec, err = cacheReporter.GetExecutionReport(executionId0) - stepResult := exec.StepResults[step1.ID] - assert.Equal(t, stepResult.ExecutionId, expectedStepResult.ExecutionId) + exec, err = runStateReporter.GetRunReport(runId0) + stepResult := exec.StepResults[stepRunId0.String()] + assert.Equal(t, stepResult.RunId, expectedStepResult.RunId) assert.Equal(t, stepResult.StepId, expectedStepResult.StepId) assert.Equal(t, stepResult.Started, expectedStepResult.Started) assert.Equal(t, stepResult.Ended, expectedStepResult.Ended) @@ -661,7 +673,7 @@ func TestReportStepStartCommandsEncoding(t *testing.T) { func TestReportStepStartManualCommand(t *testing.T) { mock_time := new(mock_time.MockTime) - cacheReporter := New(mock_time, 10) + runStateReporter := New(mock_time, 10) expectedCommand := cacao.Command{ Type: "manual", @@ -719,38 +731,41 @@ func TestReportStepStartManualCommand(t *testing.T) { Workflow: map[string]cacao.Step{step1.ID: step1, end.ID: end}, } - executionId0 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c0") + runId0 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c0") + stepRunId0 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c9") + metadata0 := run.Metadata{RunId: runId0, StepId: step1.ID, StepRunId: stepRunId0} layout := "2006-01-02T15:04:05.000Z" str := "2014-11-12T11:45:26.371Z" timeNow, _ := time.Parse(layout, str) mock_time.On("Now").Return(timeNow) - err := cacheReporter.ReportWorkflowStart(executionId0, playbook, mock_time.Now()) + err := runStateReporter.ReportWorkflowStart(runId0, playbook, mock_time.Now()) if err != nil { t.Fail() } - err = cacheReporter.ReportStepStart(executionId0, step1, cacao.NewVariables(expectedVariables), mock_time.Now()) + err = runStateReporter.ReportStepStart(metadata0, step1, cacao.NewVariables(expectedVariables), mock_time.Now()) if err != nil { t.Fail() } encodedCommand := b64.StdEncoding.EncodeToString([]byte(expectedCommand.Command)) - expectedStepStatus := cache_model.StepResult{ - ExecutionId: executionId0, + expectedStepStatus := runstate_model.StepResult{ + RunId: runId0, StepId: step1.ID, + StepRunId: stepRunId0, Started: timeNow, Ended: time.Time{}, Variables: cacao.NewVariables(expectedVariables), - Status: cache_model.Ongoing, + Status: runstate_model.Ongoing, CommandsB64: []string{encodedCommand}, Error: nil, IsAutomated: false, } - exec, err := cacheReporter.GetExecutionReport(executionId0) - stepStatus := exec.StepResults[step1.ID] - assert.Equal(t, stepStatus.ExecutionId, expectedStepStatus.ExecutionId) + exec, err := runStateReporter.GetRunReport(runId0) + stepStatus := exec.StepResults[stepRunId0.String()] + assert.Equal(t, stepStatus.RunId, expectedStepStatus.RunId) assert.Equal(t, stepStatus.StepId, expectedStepStatus.StepId) assert.Equal(t, stepStatus.Started, expectedStepStatus.Started) assert.Equal(t, stepStatus.Ended, expectedStepStatus.Ended) @@ -761,24 +776,25 @@ func TestReportStepStartManualCommand(t *testing.T) { assert.Equal(t, stepStatus.IsAutomated, expectedStepStatus.IsAutomated) assert.Equal(t, err, nil) - err = cacheReporter.ReportStepEnd(executionId0, step1, cacao.NewVariables(expectedVariables), nil, mock_time.Now()) + err = runStateReporter.ReportStepEnd(metadata0, step1, cacao.NewVariables(expectedVariables), nil, mock_time.Now()) if err != nil { t.Fail() } - expectedStepResult := cache_model.StepResult{ - ExecutionId: executionId0, - StepId: step1.ID, - Started: timeNow, - Ended: timeNow, - Variables: cacao.NewVariables(expectedVariables), - Status: cache_model.SuccessfullyExecuted, - Error: nil, + expectedStepResult := runstate_model.StepResult{ + RunId: runId0, + StepId: step1.ID, + StepRunId: stepRunId0, + Started: timeNow, + Ended: timeNow, + Variables: cacao.NewVariables(expectedVariables), + Status: runstate_model.SuccessfullyExecuted, + Error: nil, } - exec, err = cacheReporter.GetExecutionReport(executionId0) - stepResult := exec.StepResults[step1.ID] - assert.Equal(t, stepResult.ExecutionId, expectedStepResult.ExecutionId) + exec, err = runStateReporter.GetRunReport(runId0) + stepResult := exec.StepResults[stepRunId0.String()] + assert.Equal(t, stepResult.RunId, expectedStepResult.RunId) assert.Equal(t, stepResult.StepId, expectedStepResult.StepId) assert.Equal(t, stepResult.Started, expectedStepResult.Started) assert.Equal(t, stepResult.Ended, expectedStepResult.Ended) @@ -791,7 +807,7 @@ func TestReportStepStartManualCommand(t *testing.T) { func TestInvalidStepReportAfterStepEnd(t *testing.T) { mock_time := new(mock_time.MockTime) - cacheReporter := New(mock_time, 10) + runStateReporter := New(mock_time, 10) expectedCommand := cacao.Command{ Type: "ssh", @@ -849,26 +865,27 @@ func TestInvalidStepReportAfterStepEnd(t *testing.T) { Workflow: map[string]cacao.Step{step1.ID: step1, end.ID: end}, } - executionId0 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c0") + runId0 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c0") + metadata0 := run.Metadata{RunId: runId0, StepId: step1.ID, StepRunId: uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c9")} layout := "2006-01-02T15:04:05.000Z" str := "2014-11-12T11:45:26.371Z" timeNow, _ := time.Parse(layout, str) mock_time.On("Now").Return(timeNow) - err := cacheReporter.ReportWorkflowStart(executionId0, playbook, mock_time.Now()) + err := runStateReporter.ReportWorkflowStart(runId0, playbook, mock_time.Now()) if err != nil { t.Fail() } - err = cacheReporter.ReportStepStart(executionId0, step1, cacao.NewVariables(expectedVariables), mock_time.Now()) + err = runStateReporter.ReportStepStart(metadata0, step1, cacao.NewVariables(expectedVariables), mock_time.Now()) if err != nil { t.Fail() } - err = cacheReporter.ReportStepEnd(executionId0, step1, cacao.NewVariables(expectedVariables), nil, mock_time.Now()) + err = runStateReporter.ReportStepEnd(metadata0, step1, cacao.NewVariables(expectedVariables), nil, mock_time.Now()) if err != nil { t.Fail() } - err = cacheReporter.ReportStepEnd(executionId0, step1, cacao.NewVariables(expectedVariables), nil, mock_time.Now()) + err = runStateReporter.ReportStepEnd(metadata0, step1, cacao.NewVariables(expectedVariables), nil, mock_time.Now()) if err == nil { t.Fail() } @@ -878,10 +895,10 @@ func TestInvalidStepReportAfterStepEnd(t *testing.T) { mock_time.AssertExpectations(t) } -func TestAcceptedStepReportAfterExecutionEnd(t *testing.T) { +func TestAcceptedStepReportAfterRunEnd(t *testing.T) { mock_time := new(mock_time.MockTime) - cacheReporter := New(mock_time, 10) + runStateReporter := New(mock_time, 10) expectedCommand := cacao.Command{ Type: "ssh", @@ -939,26 +956,27 @@ func TestAcceptedStepReportAfterExecutionEnd(t *testing.T) { Workflow: map[string]cacao.Step{step1.ID: step1, end.ID: end}, } - executionId0 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c0") + runId0 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c0") + metadata0 := run.Metadata{RunId: runId0, StepId: step1.ID, StepRunId: uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c9")} layout := "2006-01-02T15:04:05.000Z" str := "2014-11-12T11:45:26.371Z" timeNow, _ := time.Parse(layout, str) mock_time.On("Now").Return(timeNow) - err := cacheReporter.ReportWorkflowStart(executionId0, playbook, mock_time.Now()) + err := runStateReporter.ReportWorkflowStart(runId0, playbook, mock_time.Now()) if err != nil { t.Fail() } - err = cacheReporter.ReportStepStart(executionId0, step1, cacao.NewVariables(expectedVariables), mock_time.Now()) + err = runStateReporter.ReportStepStart(metadata0, step1, cacao.NewVariables(expectedVariables), mock_time.Now()) if err != nil { t.Fail() } - err = cacheReporter.ReportWorkflowEnd(executionId0, playbook, nil, mock_time.Now()) + err = runStateReporter.ReportWorkflowEnd(runId0, playbook, nil, mock_time.Now()) if err != nil { t.Fail() } - err = cacheReporter.ReportStepEnd(executionId0, step1, cacao.NewVariables(expectedVariables), nil, mock_time.Now()) + err = runStateReporter.ReportStepEnd(metadata0, step1, cacao.NewVariables(expectedVariables), nil, mock_time.Now()) if err != nil { t.Fail() } diff --git a/internal/reporting/reporter/downstream_reporter/runstate/store.go b/internal/reporting/reporter/downstream_reporter/runstate/store.go new file mode 100644 index 00000000..502375f2 --- /dev/null +++ b/internal/reporting/reporter/downstream_reporter/runstate/store.go @@ -0,0 +1,290 @@ +package runstate + +import ( + b64 "encoding/base64" + "errors" + "fmt" + "slices" + run "soarca/internal/runs/model" + runstate_report "soarca/internal/runs/state" + "soarca/pkg/cacao" + itime "soarca/pkg/utils/time" + "sync" + "time" + + "github.com/google/uuid" +) + +const MaxRuns int = 10 + +type RunState struct { + Size int + timeUtil itime.ITime + runs map[string]runstate_report.RunEntry + fifoRegister []string + mutex sync.Mutex +} + +func New(timeUtil itime.ITime, maxRuns int) *RunState { + return &RunState{ + Size: maxRuns, + runs: make(map[string]runstate_report.RunEntry), + timeUtil: timeUtil, + mutex: sync.Mutex{}, + } +} + +// ############################### Atomic runstate access operations (mutex-protection) + +func (runStateReporter *RunState) getAllRuns() ([]runstate_report.RunEntry, error) { + runs := make([]runstate_report.RunEntry, 0) + // NOTE: fetched via fifo register key reference as is ordered array, + // this is needed to test and report back ordered runs stored + + // Lock + runStateReporter.mutex.Lock() + defer runStateReporter.mutex.Unlock() + for _, runEntryKey := range runStateReporter.fifoRegister { + // NOTE: stored runs are passed by reference, so they must not be modified + entry, ok := runStateReporter.runs[runEntryKey] + if !ok { + // Unlock + return []runstate_report.RunEntry{}, errors.New("internal error. runstate fifo register and runstate runs mismatch") + } + runs = append(runs, entry) + } + + // Unlocked + return runs, nil +} + +func (runStateReporter *RunState) getRun(runKey uuid.UUID) (runstate_report.RunEntry, error) { + + runKeyStr := runKey.String() + // No need for mutex as is one-line access + runEntry, ok := runStateReporter.runs[runKeyStr] + + if !ok { + err := errors.New("run is not in runstate. consider increasing runstate size") + return runstate_report.RunEntry{}, err + // TODO Retrieve from database and push to runstate + } + return runEntry, nil +} + +// Adding runs in FIFO logic +func (runStateReporter *RunState) addRunFIFO(newRunEntry runstate_report.RunEntry) error { + + if len(runStateReporter.fifoRegister) != len(runStateReporter.runs) { + return errors.New("runstate fifo register and content are desynchronized") + } + + newRunEntryKey := newRunEntry.RunId.String() + + // Lock + runStateReporter.mutex.Lock() + defer runStateReporter.mutex.Unlock() + + if _, ok := runStateReporter.runs[newRunEntryKey]; ok { + return errors.New("there is already an run in the runstate with the same run id") + } + if len(runStateReporter.fifoRegister) >= runStateReporter.Size { + + firstRun := runStateReporter.fifoRegister[0] + runStateReporter.fifoRegister = runStateReporter.fifoRegister[1:] + delete(runStateReporter.runs, firstRun) + runStateReporter.fifoRegister = append(runStateReporter.fifoRegister, newRunEntryKey) + runStateReporter.runs[newRunEntryKey] = newRunEntry + + return nil + // Unlocked + } + runStateReporter.fifoRegister = append(runStateReporter.fifoRegister, newRunEntryKey) + runStateReporter.runs[newRunEntryKey] = newRunEntry + + return nil + // Unlocked +} + +func (runStateReporter *RunState) upateEndRunWorkflow(runId uuid.UUID, workflowError error, at time.Time) error { + // The runstate should stay locked for the whole modification period + // in order to prevent e.g. the run data being popped-out due to FIFO + // while its status or some of its steps are being updated + + // Lock + runStateReporter.mutex.Lock() + defer runStateReporter.mutex.Unlock() + + runEntry, err := runStateReporter.getRun(runId) + if err != nil { + return err + } + + if workflowError != nil { + runEntry.Error = workflowError + runEntry.Status = runstate_report.Failed + } else { + runEntry.Status = runstate_report.SuccessfullyExecuted + } + runEntry.Ended = at + runStateReporter.runs[runId.String()] = runEntry + + return nil + // Unlocked +} + +func (runStateReporter *RunState) addStartRunStep(runId uuid.UUID, newStepData runstate_report.StepResult) error { + // Locked + runStateReporter.mutex.Lock() + defer runStateReporter.mutex.Unlock() + + runEntry, err := runStateReporter.getRun(runId) + if err != nil { + return err + } + + if runEntry.Status != runstate_report.Ongoing { + return errors.New("trying to report on the run of a step for an already reportedly terminated playbook run") + } + stepRunKey := newStepData.StepRunId.String() + _, alreadyThere := runEntry.StepResults[stepRunKey] + if alreadyThere { + // A collision here would mean the same StepRunId was minted + // twice, which should never happen - each step invocation gets a + // fresh one. Re-runs of the same StepId are expected and get + // their own distinct entry. + return errors.New("a step run start was already reported for this step run. ignoring") + } + + runEntry.StepResults[stepRunKey] = newStepData + // New code + runStateReporter.runs[runId.String()] = runEntry + + return nil + // Unlocked +} + +func (runStateReporter *RunState) upateEndRunStep(runId uuid.UUID, stepRunId uuid.UUID, returnVars cacao.Variables, stepError error, acceptedStepStati []runstate_report.Status, at time.Time) error { + // Locked + runStateReporter.mutex.Lock() + defer runStateReporter.mutex.Unlock() + + runEntry, err := runStateReporter.getRun(runId) + if err != nil { + return err + } + + stepRunKey := stepRunId.String() + runStepResult, ok := runEntry.StepResults[stepRunKey] + if !ok { + return errors.New("trying to update a step run which was not (yet?) recorded in the runstate") + // Unlocked + } + + if !slices.Contains(acceptedStepStati, runStepResult.Status) { + return fmt.Errorf("step status precondition not met for step update [step status: %s]", runStepResult.Status.String()) + } + + if stepError != nil { + runStepResult.Error = stepError + runStepResult.Status = runstate_report.ServerSideError + } else { + runStepResult.Status = runstate_report.SuccessfullyExecuted + } + runStepResult.Ended = at + runStepResult.Variables = returnVars + runEntry.StepResults[stepRunKey] = runStepResult + runStateReporter.runs[runId.String()] = runEntry + + return nil + // Unlocked +} + +// Run-state query interface + +func (runStateReporter *RunState) GetRuns() ([]runstate_report.RunEntry, error) { + runs, err := runStateReporter.getAllRuns() + return runs, err +} + +func (runStateReporter *RunState) GetRunReport(runKey uuid.UUID) (runstate_report.RunEntry, error) { + + runEntry, err := runStateReporter.getRun(runKey) + if err != nil { + return runstate_report.RunEntry{}, err + } + + return runEntry, nil +} + +// ############################### Reporting interface + +func (runStateReporter *RunState) ReportWorkflowStart(runId uuid.UUID, playbook cacao.Playbook, at time.Time) error { + + newRunEntry := runstate_report.RunEntry{ + RunId: runId, + PlaybookId: playbook.ID, + Name: playbook.Name, + Description: playbook.Description, + Started: at, + Ended: time.Time{}, + StepResults: map[string]runstate_report.StepResult{}, + Status: runstate_report.Ongoing, + } + err := runStateReporter.addRunFIFO(newRunEntry) + if err != nil { + return err + } + return nil +} + +func (runStateReporter *RunState) ReportWorkflowEnd(runId uuid.UUID, playbook cacao.Playbook, workflowError error, at time.Time) error { + + err := runStateReporter.upateEndRunWorkflow(runId, workflowError, at) + return err +} + +func (runStateReporter *RunState) ReportStepStart(metadata run.Metadata, step cacao.Step, variables cacao.Variables, at time.Time) error { + + commandsB64 := []string{} + isAutomated := true + for _, cmd := range step.Commands { + if cmd.Type == cacao.CommandTypeManual { + isAutomated = false + } + if cmd.CommandB64 != "" { + commandsB64 = append(commandsB64, cmd.CommandB64) + } else { + cmdB64 := b64.StdEncoding.EncodeToString([]byte(cmd.Command)) + commandsB64 = append(commandsB64, cmdB64) + } + } + + newStep := runstate_report.StepResult{ + RunId: metadata.RunId, + StepId: step.ID, + StepRunId: metadata.StepRunId, + Name: step.Name, + Description: step.Description, + //Started: runStateReporter.timeUtil.Now(), + Started: at, + Ended: time.Time{}, + Variables: variables, + CommandsB64: commandsB64, + Status: runstate_report.Ongoing, + Error: nil, + IsAutomated: isAutomated, + } + + err := runStateReporter.addStartRunStep(metadata.RunId, newStep) + + return err +} + +func (runStateReporter *RunState) ReportStepEnd(metadata run.Metadata, step cacao.Step, returnVars cacao.Variables, stepError error, at time.Time) error { + + acceptedStepStati := []runstate_report.Status{runstate_report.Ongoing} + err := runStateReporter.upateEndRunStep(metadata.RunId, metadata.StepRunId, returnVars, stepError, acceptedStepStati, at) + + return err +} diff --git a/internal/reporting/reporter/downstream_reporter/sink.go b/internal/reporting/reporter/downstream_reporter/sink.go new file mode 100644 index 00000000..126e1f83 --- /dev/null +++ b/internal/reporting/reporter/downstream_reporter/sink.go @@ -0,0 +1,20 @@ +package downstream_reporter + +import ( + "soarca/pkg/cacao" + "soarca/internal/runs/model" + "time" + + "github.com/google/uuid" +) + +type IDownStreamReporter interface { + ReportWorkflowStart(runId uuid.UUID, playbook cacao.Playbook, at time.Time) error + ReportWorkflowEnd(runId uuid.UUID, playbook cacao.Playbook, err error, at time.Time) error + + // metadata.StepRunId identifies this specific invocation of + // metadata.StepId - see run.Metadata for why StepId alone is not + // sufficient. + ReportStepStart(metadata run.Metadata, step cacao.Step, stepResults cacao.Variables, at time.Time) error + ReportStepEnd(metadata run.Metadata, step cacao.Step, stepResults cacao.Variables, err error, at time.Time) error +} diff --git a/pkg/reporting/reporter/reporter_test.go b/internal/reporting/reporter/reporter_test.go similarity index 78% rename from pkg/reporting/reporter/reporter_test.go rename to internal/reporting/reporter/reporter_test.go index d436e2a9..2573b4bd 100644 --- a/pkg/reporting/reporter/reporter_test.go +++ b/internal/reporting/reporter/reporter_test.go @@ -1,8 +1,9 @@ package reporter import ( - "soarca/pkg/models/cacao" - ds_reporter "soarca/pkg/reporting/reporter/downstream_reporter" + "soarca/pkg/cacao" + "soarca/internal/runs/model" + ds_reporter "soarca/internal/reporting/reporter/downstream_reporter" "soarca/test/unittest/mocks/mock_reporter" mock_time "soarca/test/unittest/mocks/mock_utils/time" "sync" @@ -99,7 +100,7 @@ func TestReportWorkflowStart(t *testing.T) { Workflow: map[string]cacao.Step{step1.ID: step1, end.ID: end}, } - executionId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + runId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") layout := "2006-01-02T15:04:05.000Z" str := "2014-11-12T11:45:26.371Z" @@ -107,8 +108,8 @@ func TestReportWorkflowStart(t *testing.T) { mock_time.On("Now").Return(timeNow) wg.Add(1) - mock_ds_reporter.On("ReportWorkflowStart", executionId, playbook, timeNow).Return(nil) - reporter.ReportWorkflowStart(executionId, playbook, mock_time.Now()) + mock_ds_reporter.On("ReportWorkflowStart", runId, playbook, timeNow).Return(nil) + reporter.ReportWorkflowStart(runId, playbook, mock_time.Now()) wg.Wait() mock_ds_reporter.AssertExpectations(t) @@ -178,7 +179,7 @@ func TestReportWorkflowEnd(t *testing.T) { Workflow: map[string]cacao.Step{step1.ID: step1, end.ID: end}, } - executionId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + runId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") layout := "2006-01-02T15:04:05.000Z" str := "2014-11-12T11:45:26.371Z" @@ -186,8 +187,8 @@ func TestReportWorkflowEnd(t *testing.T) { mock_time.On("Now").Return(timeNow) wg.Add(1) - mock_ds_reporter.On("ReportWorkflowEnd", executionId, playbook, nil, timeNow).Return(nil) - reporter.ReportWorkflowEnd(executionId, playbook, nil, mock_time.Now()) + mock_ds_reporter.On("ReportWorkflowEnd", runId, playbook, nil, timeNow).Return(nil) + reporter.ReportWorkflowEnd(runId, playbook, nil, mock_time.Now()) wg.Wait() mock_ds_reporter.AssertExpectations(t) @@ -223,7 +224,8 @@ func TestReportStepStart(t *testing.T) { Targets: []string{"target1"}, } - executionId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + runId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + metadata := run.Metadata{RunId: runId, StepId: step1.ID} layout := "2006-01-02T15:04:05.000Z" str := "2014-11-12T11:45:26.371Z" @@ -231,8 +233,8 @@ func TestReportStepStart(t *testing.T) { mock_time.On("Now").Return(timeNow) wg.Add(1) - mock_ds_reporter.On("ReportStepStart", executionId, step1, cacao.NewVariables(expectedVariables), timeNow).Return(nil) - reporter.ReportStepStart(executionId, step1, cacao.NewVariables(expectedVariables), mock_time.Now()) + mock_ds_reporter.On("ReportStepStart", metadata, step1, cacao.NewVariables(expectedVariables), timeNow).Return(nil) + reporter.ReportStepStart(metadata, step1, cacao.NewVariables(expectedVariables), mock_time.Now()) wg.Wait() mock_ds_reporter.AssertExpectations(t) @@ -267,7 +269,8 @@ func TestReportStepEnd(t *testing.T) { Targets: []string{"target1"}, } - executionId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + runId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + metadata := run.Metadata{RunId: runId, StepId: step1.ID} layout := "2006-01-02T15:04:05.000Z" str := "2014-11-12T11:45:26.371Z" @@ -275,8 +278,8 @@ func TestReportStepEnd(t *testing.T) { mock_time.On("Now").Return(timeNow) wg.Add(1) - mock_ds_reporter.On("ReportStepEnd", executionId, step1, cacao.NewVariables(expectedVariables), nil, timeNow).Return(nil) - reporter.ReportStepEnd(executionId, step1, cacao.NewVariables(expectedVariables), nil, mock_time.Now()) + mock_ds_reporter.On("ReportStepEnd", metadata, step1, cacao.NewVariables(expectedVariables), nil, timeNow).Return(nil) + reporter.ReportStepEnd(metadata, step1, cacao.NewVariables(expectedVariables), nil, mock_time.Now()) wg.Wait() mock_ds_reporter.AssertExpectations(t) @@ -347,7 +350,8 @@ func TestMultipleDownstreamReporters(t *testing.T) { Workflow: map[string]cacao.Step{step1.ID: step1, end.ID: end}, } - executionId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + runId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + metadata := run.Metadata{RunId: runId, StepId: step1.ID} layout := "2006-01-02T15:04:05.000Z" str := "2014-11-12T11:45:26.371Z" @@ -355,25 +359,25 @@ func TestMultipleDownstreamReporters(t *testing.T) { mock_time.On("Now").Return(timeNow) wg.Add(2) - mock_ds_reporter1.On("ReportWorkflowStart", executionId, playbook, timeNow).Return(nil) - mock_ds_reporter2.On("ReportWorkflowStart", executionId, playbook, timeNow).Return(nil) + mock_ds_reporter1.On("ReportWorkflowStart", runId, playbook, timeNow).Return(nil) + mock_ds_reporter2.On("ReportWorkflowStart", runId, playbook, timeNow).Return(nil) wg.Add(2) - mock_ds_reporter1.On("ReportStepStart", executionId, step1, cacao.NewVariables(expectedVariables), timeNow).Return(nil) - mock_ds_reporter2.On("ReportStepStart", executionId, step1, cacao.NewVariables(expectedVariables), timeNow).Return(nil) + mock_ds_reporter1.On("ReportStepStart", metadata, step1, cacao.NewVariables(expectedVariables), timeNow).Return(nil) + mock_ds_reporter2.On("ReportStepStart", metadata, step1, cacao.NewVariables(expectedVariables), timeNow).Return(nil) wg.Add(2) - mock_ds_reporter1.On("ReportStepEnd", executionId, step1, cacao.NewVariables(expectedVariables), nil, timeNow).Return(nil) - mock_ds_reporter2.On("ReportStepEnd", executionId, step1, cacao.NewVariables(expectedVariables), nil, timeNow).Return(nil) + mock_ds_reporter1.On("ReportStepEnd", metadata, step1, cacao.NewVariables(expectedVariables), nil, timeNow).Return(nil) + mock_ds_reporter2.On("ReportStepEnd", metadata, step1, cacao.NewVariables(expectedVariables), nil, timeNow).Return(nil) wg.Add(2) - mock_ds_reporter1.On("ReportWorkflowEnd", executionId, playbook, nil, timeNow).Return(nil) - mock_ds_reporter2.On("ReportWorkflowEnd", executionId, playbook, nil, timeNow).Return(nil) + mock_ds_reporter1.On("ReportWorkflowEnd", runId, playbook, nil, timeNow).Return(nil) + mock_ds_reporter2.On("ReportWorkflowEnd", runId, playbook, nil, timeNow).Return(nil) - reporter.ReportWorkflowStart(executionId, playbook, mock_time.Now()) - reporter.ReportStepStart(executionId, step1, cacao.NewVariables(expectedVariables), mock_time.Now()) - reporter.ReportStepEnd(executionId, step1, cacao.NewVariables(expectedVariables), nil, mock_time.Now()) - reporter.ReportWorkflowEnd(executionId, playbook, nil, mock_time.Now()) + reporter.ReportWorkflowStart(runId, playbook, mock_time.Now()) + reporter.ReportStepStart(metadata, step1, cacao.NewVariables(expectedVariables), mock_time.Now()) + reporter.ReportStepEnd(metadata, step1, cacao.NewVariables(expectedVariables), nil, mock_time.Now()) + reporter.ReportWorkflowEnd(runId, playbook, nil, mock_time.Now()) wg.Wait() mock_ds_reporter1.AssertExpectations(t) diff --git a/pkg/reporting/reporter/reporter.go b/internal/reporting/reporter/workflow.go similarity index 64% rename from pkg/reporting/reporter/reporter.go rename to internal/reporting/reporter/workflow.go index 8127d961..a177d06e 100644 --- a/pkg/reporting/reporter/reporter.go +++ b/internal/reporting/reporter/workflow.go @@ -8,8 +8,9 @@ import ( "time" "soarca/internal/logger" - "soarca/pkg/models/cacao" - downstreamReporter "soarca/pkg/reporting/reporter/downstream_reporter" + "soarca/pkg/cacao" + "soarca/internal/runs/model" + downstreamReporter "soarca/internal/reporting/reporter/downstream_reporter" "soarca/pkg/utils" "github.com/google/uuid" @@ -27,13 +28,17 @@ func init() { // Reporter interfaces type IWorkflowReporter interface { // -> Give info to downstream reporters - ReportWorkflowStart(executionId uuid.UUID, playbook cacao.Playbook, at time.Time) - ReportWorkflowEnd(executionId uuid.UUID, playbook cacao.Playbook, workflowError error, at time.Time) + ReportWorkflowStart(runId uuid.UUID, playbook cacao.Playbook, at time.Time) + ReportWorkflowEnd(runId uuid.UUID, playbook cacao.Playbook, workflowError error, at time.Time) } type IStepReporter interface { // -> Give info to downstream reporters - ReportStepStart(executionId uuid.UUID, step cacao.Step, returnVars cacao.Variables, at time.Time) - ReportStepEnd(executionId uuid.UUID, step cacao.Step, returnVars cacao.Variables, stepError error, at time.Time) + // + // metadata.StepRunId identifies this specific invocation of + // metadata.StepId, so downstream reporters can key per-invocation data + // (e.g. re-executed steps in a while-loop) without collisions. + ReportStepStart(metadata run.Metadata, step cacao.Step, returnVars cacao.Variables, at time.Time) + ReportStepEnd(metadata run.Metadata, step cacao.Step, returnVars cacao.Variables, stepError error, at time.Time) } const MaxReporters int = 10 @@ -86,13 +91,13 @@ func (reporter *Reporter) RegisterReporters(reporters []downstreamReporter.IDown // ######################## IWorkflowReporter interface -func (reporter *Reporter) ReportWorkflowStart(executionId uuid.UUID, playbook cacao.Playbook, at time.Time) { - log.Trace(fmt.Sprintf("[execution: %s, playbook: %s] reporting workflow start", executionId, playbook.ID)) +func (reporter *Reporter) ReportWorkflowStart(runId uuid.UUID, playbook cacao.Playbook, at time.Time) { + log.Trace(fmt.Sprintf("[run: %s, playbook: %s] reporting workflow start", runId, playbook.ID)) reporter.wg.Add(1) reporter.reportingch <- func() { defer reporter.wg.Done() for _, downstreamRep := range reporter.reporters { - err := downstreamRep.ReportWorkflowStart(executionId, playbook, at) + err := downstreamRep.ReportWorkflowStart(runId, playbook, at) if err != nil { log.Trace("reportWorkflowStart error") log.Warning(err) @@ -101,13 +106,13 @@ func (reporter *Reporter) ReportWorkflowStart(executionId uuid.UUID, playbook ca } } -func (reporter *Reporter) ReportWorkflowEnd(executionId uuid.UUID, playbook cacao.Playbook, workflowError error, at time.Time) { - log.Trace(fmt.Sprintf("[execution: %s, playbook: %s] reporting workflow end", executionId, playbook.ID)) +func (reporter *Reporter) ReportWorkflowEnd(runId uuid.UUID, playbook cacao.Playbook, workflowError error, at time.Time) { + log.Trace(fmt.Sprintf("[run: %s, playbook: %s] reporting workflow end", runId, playbook.ID)) reporter.wg.Add(1) reporter.reportingch <- func() { defer reporter.wg.Done() for _, downstreamRep := range reporter.reporters { - err := downstreamRep.ReportWorkflowEnd(executionId, playbook, workflowError, at) + err := downstreamRep.ReportWorkflowEnd(runId, playbook, workflowError, at) if err != nil { log.Trace("reportWorkflowEnd error") log.Warning(err) @@ -118,13 +123,13 @@ func (reporter *Reporter) ReportWorkflowEnd(executionId uuid.UUID, playbook caca // ######################## IStepReporter interface -func (reporter *Reporter) ReportStepStart(executionId uuid.UUID, step cacao.Step, returnVars cacao.Variables, at time.Time) { - log.Trace(fmt.Sprintf("[execution: %s, step: %s] reporting step start", executionId, step.ID)) +func (reporter *Reporter) ReportStepStart(metadata run.Metadata, step cacao.Step, returnVars cacao.Variables, at time.Time) { + log.Trace(fmt.Sprintf("[run: %s, step: %s, step-run: %s] reporting step start", metadata.RunId, step.ID, metadata.StepRunId)) reporter.wg.Add(1) reporter.reportingch <- func() { defer reporter.wg.Done() for _, downstreamRep := range reporter.reporters { - err := downstreamRep.ReportStepStart(executionId, step, returnVars, at) + err := downstreamRep.ReportStepStart(metadata, step, returnVars, at) if err != nil { log.Trace("reportStepStart error") log.Warning(err) @@ -133,13 +138,13 @@ func (reporter *Reporter) ReportStepStart(executionId uuid.UUID, step cacao.Step } } -func (reporter *Reporter) ReportStepEnd(executionId uuid.UUID, step cacao.Step, returnVars cacao.Variables, stepError error, at time.Time) { - log.Trace(fmt.Sprintf("[execution: %s, step: %s] reporting step end", executionId, step.ID)) +func (reporter *Reporter) ReportStepEnd(metadata run.Metadata, step cacao.Step, returnVars cacao.Variables, stepError error, at time.Time) { + log.Trace(fmt.Sprintf("[run: %s, step: %s, step-run: %s] reporting step end", metadata.RunId, step.ID, metadata.StepRunId)) reporter.wg.Add(1) reporter.reportingch <- func() { defer reporter.wg.Done() for _, downstreamRep := range reporter.reporters { - err := downstreamRep.ReportStepEnd(executionId, step, returnVars, stepError, at) + err := downstreamRep.ReportStepEnd(metadata, step, returnVars, stepError, at) if err != nil { log.Trace("reportStepEnd error") log.Warning(err) diff --git a/internal/runs/engine/factory.go b/internal/runs/engine/factory.go new file mode 100644 index 00000000..9ecd50c9 --- /dev/null +++ b/internal/runs/engine/factory.go @@ -0,0 +1,147 @@ +// Package engine builds the per-run workflow walker and owns all capability, +// executor and reporter wiring. +package engine + +import ( + "reflect" + "time" + + "soarca/internal/config" + "soarca/internal/logger" + "soarca/internal/store" + "soarca/internal/workflow" + "soarca/internal/workflow/capability" + fincap "soarca/internal/workflow/capability/fin" + "soarca/internal/workflow/capability/fin/queue" + httpcap "soarca/internal/workflow/capability/http" + manualcap "soarca/internal/workflow/capability/manual" + "soarca/internal/workflow/capability/manual/inbox" + openc2cap "soarca/internal/workflow/capability/openc2" + pscap "soarca/internal/workflow/capability/powershell" + sshcap "soarca/internal/workflow/capability/ssh" + actionexec "soarca/internal/workflow/steps/action" + condexec "soarca/internal/workflow/steps/condition" + pbactionexec "soarca/internal/workflow/steps/playbook_action" + "soarca/pkg/extensions/soarca/assignment" + thehivecases "soarca/internal/adapters/thehive/cases" + thehiveconnector "soarca/internal/adapters/thehive/common/connector" + thehivereport "soarca/internal/adapters/thehive/reporter" + "soarca/internal/reporting/cases" + "soarca/internal/reporting/reporter" + downstreamreport "soarca/internal/reporting/reporter/downstream_reporter" + "soarca/pkg/utils/guid" + httputil "soarca/pkg/utils/http" + stixcmp "soarca/pkg/utils/stix/expression/comparison" + timeutil "soarca/pkg/utils/time" +) + +var log *logger.Log + +type empty struct{} + +func init() { + log = logger.Logger(reflect.TypeOf(empty{}).PkgPath(), logger.Info, "", logger.Json) +} + +// Deps are the collaborators a walker needs, passed explicitly so the +// engine never depends on the runtime container. +type Deps struct { + ManualInbox inbox.Dispatcher + RunState downstreamreport.IDownStreamReporter + FinQueue *queue.Queue + FinStore storage.FinStore + PlaybookStore storage.PlaybookStore + SkipCertValidation bool + FinStaleAfter time.Duration + TheHive config.TheHiveConfig +} + +// Factory creates a workflow walker per playbook run. +type Factory struct { + deps Deps +} + +func New(deps Deps) *Factory { + return &Factory{deps: deps} +} + +// NewWalker builds a workflow walker for a single run. +func (f *Factory) NewWalker() workflow.Walker { + sshCap := new(sshcap.SshCapability) + capabilities := map[string]capability.ICapability{sshCap.GetType(): sshCap} + + httpUtil := new(httputil.HttpRequest) + httpUtil.SkipCertificateValidation(f.deps.SkipCertValidation) + httpCap := httpcap.New(httpUtil) + capabilities[httpCap.GetType()] = httpCap + + openc2Cap := openc2cap.New(httpUtil) + capabilities[openc2Cap.GetType()] = openc2Cap + + powershellCap := pscap.New() + capabilities[powershellCap.GetType()] = powershellCap + + man := manualcap.New(f.deps.ManualInbox) + capabilities[man.GetType()] = &man + + report := reporter.New([]downstreamreport.IDownStreamReporter{}) + downstreamReporters := []downstreamreport.IDownStreamReporter{f.deps.RunState} + + thehiveReporter, theHiveCaseManager := f.initializeTheHiveReporting() + if thehiveReporter != nil { + downstreamReporters = append(downstreamReporters, thehiveReporter) + } + + report.RegisterReporters(downstreamReporters) + + soarcaTime := new(timeutil.Time) + assignmentExt := assignment.New() + actionExec := actionexec.New(capabilities, report, soarcaTime, assignmentExt) + + actionExec.SetFinFallback(fincap.New(fincap.Dependencies{ + Queue: f.deps.FinQueue, + GUID: new(guid.Guid), + Store: f.deps.FinStore, + Time: soarcaTime, + StaleAfter: f.deps.FinStaleAfter, + })) + + pbExec := pbactionexec.New(f.NewWalker, f.deps.PlaybookStore, report, soarcaTime) + stixCmp := stixcmp.New() + condExec := condexec.New(stixCmp, report, soarcaTime) + guidGen := new(guid.Guid) + + return workflow.New(actionExec, pbExec, condExec, guidGen, report, soarcaTime, theHiveCaseManager) +} + +// initializeTheHiveReporting sets up The Hive integration if configured. +func (f *Factory) initializeTheHiveReporting() (downstreamreport.IDownStreamReporter, cases.ICasesManager) { + cfg := f.deps.TheHive + if !cfg.Activate { + return nil, nil + } + + log.Info("Initializing The Hive reporting integration") + + if len(cfg.APIBaseURL) < 1 || len(cfg.APIToken) < 1 { + log.Warning("Could not initialize The Hive reporting integration. Check environment variables.") + return nil, nil + } + + log.Infof("Creating The Hive connector with API base URL: %s", cfg.APIBaseURL) + conn := thehiveconnector.NewConnector(cfg.APIBaseURL, cfg.APIToken, cfg.AllowInsecure) + + if cfg.EnableCaseManager { + log.Info("Enabling The Hive case manager") + caseMgr := thehivecases.NewCaseManager(conn) + return caseMgr, caseMgr + } + + if cfg.EnableReporter { + log.Info("Enabling The Hive reporter") + rep := thehivereport.NewReporter(conn) + return rep, nil + } + + return nil, nil +} diff --git a/internal/runs/model/metadata.go b/internal/runs/model/metadata.go new file mode 100644 index 00000000..f0a1812a --- /dev/null +++ b/internal/runs/model/metadata.go @@ -0,0 +1,20 @@ +package run + +import ( + "github.com/google/uuid" +) + +type Metadata struct { + RunId uuid.UUID + PlaybookId string + StepId string + // StepRunId uniquely identifies one *invocation* of StepId within + // RunId. StepId alone is not enough: while-loops and cyclic + // on_completion graphs can visit the same StepId more than once within a + // single run, and (once StepTypeParallel is implemented) the same + // StepId could even be dispatched concurrently. A fresh StepRunId + // is minted for every such invocation, so per-invocation state (pending + // pending manual commands, run-state entries, ...) can be keyed + // without colliding across re-runs of the same step. + StepRunId uuid.UUID +} diff --git a/internal/runs/service.go b/internal/runs/service.go new file mode 100644 index 00000000..e23e0458 --- /dev/null +++ b/internal/runs/service.go @@ -0,0 +1,146 @@ +// Package runs owns playbook runs: starting them, and reading back their +// recorded state. +package runs + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/google/uuid" + + "soarca/internal/store" + "soarca/internal/workflow" + "soarca/pkg/cacao" + "soarca/internal/runs/state" +) + +// Runner is the run use case surface offered to any driver +// (HTTP, gRPC, CLI, embedded SDK). +type Runner interface { + // Start runs a playbook supplied by the caller. + Start(ctx context.Context, playbook *cacao.Playbook, variables cacao.Variables) (runID uuid.UUID, err error) + + // StartByID runs a stored playbook. + StartByID(ctx context.Context, playbookID string, variables cacao.Variables) (runID uuid.UUID, err error) + + // List returns all recorded run summaries. + List(ctx context.Context) ([]runstate.RunEntry, error) + + // Report returns the detailed report for a single run. + Report(ctx context.Context, runID uuid.UUID) (runstate.RunEntry, error) +} + +// Reports reads recorded run state. +type Reports interface { + GetRuns() ([]runstate.RunEntry, error) + GetRunReport(runID uuid.UUID) (runstate.RunEntry, error) +} + +// ValidationError marks a rejected run request. +type ValidationError struct { + Err error +} + +func (e ValidationError) Error() string { + return e.Err.Error() +} + +func (e ValidationError) Unwrap() error { + return e.Err +} + +// Service implements Runner. +type Service struct { + newWalker workflow.NewWalker + playbooks storage.PlaybookStore + reports Reports +} + +func New(newWalker workflow.NewWalker, playbooks storage.PlaybookStore, reports Reports) *Service { + return &Service{ + newWalker: newWalker, + playbooks: playbooks, + reports: reports, + } +} + +// StartByID loads a stored playbook, applies the supplied variables and starts a run. +func (s *Service) StartByID(ctx context.Context, playbookID string, variables cacao.Variables) (uuid.UUID, error) { + playbook, err := s.playbooks.Get(ctx, playbookID) + if err != nil { + return uuid.Nil, err + } + return s.Start(ctx, &playbook, variables) +} + +// Start applies the supplied variables to the playbook and starts a run. +func (s *Service) Start(ctx context.Context, playbook *cacao.Playbook, variables cacao.Variables) (uuid.UUID, error) { + if err := mergeVariablesInPlaybook(playbook, variables); err != nil { + return uuid.Nil, ValidationError{Err: err} + } + + walker := s.newWalker() + results := make(chan workflow.Result, 1) + + go walker.ExecuteAsync(*playbook, results) + + select { + case <-ctx.Done(): + return uuid.Nil, ctx.Err() + case r := <-results: + return r.RunId, nil + } +} + +// List returns all recorded run summaries. +func (s *Service) List(ctx context.Context) ([]runstate.RunEntry, error) { + _ = ctx + return s.reports.GetRuns() +} + +// Report returns the detailed report for a single run. +func (s *Service) Report(ctx context.Context, runID uuid.UUID) (runstate.RunEntry, error) { + _ = ctx + return s.reports.GetRunReport(runID) +} + +func mergeVariablesInPlaybook(playbook *cacao.Playbook, payloadVariables cacao.Variables) error { + for name, variable := range payloadVariables { + if _, ok := playbook.PlaybookVariables[name]; !ok { + return fmt.Errorf("provided variables is not a valid subset of the variables for the referenced playbook [ playbook id: %s ]", playbook.ID) + } + if variable.Type != playbook.PlaybookVariables[name].Type { + return fmt.Errorf("mismatch in variables type for [ %s ]: payload var type = %s, playbook var type = %s", name, variable.Type, playbook.PlaybookVariables[name].Type) + } + if !playbook.PlaybookVariables[name].External { + return fmt.Errorf("playbook variable [ %s ] cannot be assigned in playbook because it is not marked as external in the plabook", name) + } + + updatedVariable := cacao.Variable{ + Name: name, + Type: playbook.PlaybookVariables[name].Type, + Description: playbook.PlaybookVariables[name].Description, + Value: variable.Value, + Constant: playbook.PlaybookVariables[name].Constant, + External: playbook.PlaybookVariables[name].External, + } + playbook.PlaybookVariables[name] = updatedVariable + } + return nil +} + +// DecodeVariables decodes a JSON run payload into variables. +func DecodeVariables(body []byte) (cacao.Variables, error) { + payloadVariables := cacao.NewVariables() + if err := json.Unmarshal(body, &payloadVariables); err != nil { + return nil, errors.New("cannot unmarshal provided variables") + } + return payloadVariables, nil +} + +// DecodePlaybook parses a playbook payload. +func DecodePlaybook(body []byte) *cacao.Playbook { + return cacao.Decode(body) +} diff --git a/pkg/models/cache/cache.go b/internal/runs/state/report.go similarity index 69% rename from pkg/models/cache/cache.go rename to internal/runs/state/report.go index 9e480f07..1e6957e6 100644 --- a/pkg/models/cache/cache.go +++ b/internal/runs/state/report.go @@ -1,7 +1,7 @@ -package cache +package runstate import ( - "soarca/pkg/models/cacao" + "soarca/pkg/cacao" "time" "github.com/google/uuid" @@ -33,27 +33,32 @@ func (status Status) String() string { }[status] } -type ExecutionEntry struct { - ExecutionId uuid.UUID +type RunEntry struct { + RunId uuid.UUID Name string Description string PlaybookId string Started time.Time Ended time.Time + // StepResults is keyed by StepRunId (not StepId): the same StepId + // can be invoked more than once within one run (while-loops, + // cyclic on_completion graphs), and each such invocation gets its own + // entry here instead of overwriting/being rejected. StepResults map[string]StepResult Error error Status Status } type StepResult struct { - ExecutionId uuid.UUID + RunId uuid.UUID StepId string + StepRunId uuid.UUID Name string Description string Started time.Time Ended time.Time // Make sure we can have a playbookID for playbook actions, and also - // the execution ID for the invoked playbook + // the run ID for the invoked playbook CommandsB64 []string Variables cacao.Variables Status Status diff --git a/internal/services/interfaces.go b/internal/services/interfaces.go new file mode 100644 index 00000000..f448836a --- /dev/null +++ b/internal/services/interfaces.go @@ -0,0 +1,133 @@ +package services + +import ( + "context" + + manual "soarca/internal/manual/model" + "soarca/internal/playbooks" + run "soarca/internal/runs/model" + "soarca/pkg/cacao" + "soarca/pkg/models/fin" + + "github.com/google/uuid" +) + +// ============================================================================ +// OPERATIONAL SERVICES +// ============================================================================ + +// FinRegistry manages FIN registration and admin lifecycle. +// +// Dependencies: FinStore (only) +// Does NOT depend on: playbook run, job leasing. +type FinRegistry interface { + // RegisterFin registers a new FIN with the given capabilities. + RegisterFin(ctx context.Context, req fin.RegisterRequest) (finID string, token string, err error) + + // UnregisterFin unregisters a FIN and cleans up its record. + UnregisterFin(ctx context.Context, finToken string) error + + // ListFins returns all registered FINs (admin use). + ListFins(ctx context.Context) (fins []fin.Record, err error) + + // GetFin retrieves a specific FIN record by ID (admin use). + GetFin(ctx context.Context, finID string) (record fin.Record, err error) + + // DeleteFin removes a FIN record (admin use). + DeleteFin(ctx context.Context, finID string) error + + // ValidateToken checks if a token is valid and returns the FIN ID. + // Used by auth middleware to verify credentials. + ValidateToken(ctx context.Context, finToken string) (finID string, err error) +} + +// FinWorkService manages leased FIN work items. +// +// Dependencies: FinQueue, FinStore (only) +// Does NOT depend on: playbook run, manual commands, HTTP. +// +// Lease semantics belong here, not in the run runtime. +type FinWorkService interface { + // PollJob polls for available work matching FIN's capabilities. + // Long-polls until a job is available or context timeout/cancellation. + PollJob(ctx context.Context, finToken string, pollReq fin.PollRequest) (job *fin.Job, err error) + + // SubmitJobResult submits the result of a claimed job. + SubmitJobResult(ctx context.Context, finToken string, jobID uuid.UUID, result fin.JobResult) error + + // HeartbeatJob extends the lease on an in-flight job (status-ping). + HeartbeatJob(ctx context.Context, finToken string, jobID uuid.UUID) error +} + +// ManualInbox manages manual step resolution during playbook run. +// Depends on: manual inbox +// Does NOT depend on: FIN leasing or claim semantics. +// +// Responsibility: tracking pending manual commands, allowing operators to +// view pending steps, and providing responses for outstanding manual work. +type ManualInbox interface { + // ListPendingCommands returns all pending manual steps across all runs. + ListPendingCommands() (commands []manual.CommandInfo, err error) + + // GetPendingCommand retrieves a specific pending manual step. + GetPendingCommand(metadata run.Metadata) (command manual.CommandInfo, err error) + + // ContinuePendingCommand resolves a pending manual step with the operator's response. + ContinuePendingCommand(response manual.Response) error +} + +// ============================================================================ +// APPLICATION SERVICES (thin orchestrators) +// ============================================================================ + +// PlaybookService provides CRUD operations over the playbook repository. +// Depends on: PlaybookStore +// +// Responsibility: playbook lifecycle (create, read, update, delete, list). +// Future: could add versioning, validation, domain rules. +type PlaybookService interface { + // ListPlaybooks returns all stored playbooks. + ListPlaybooks(ctx context.Context) (playbooks []cacao.Playbook, err error) + + // GetPlaybook retrieves a playbook by ID. + GetPlaybook(ctx context.Context, playbookID string) (playbook *cacao.Playbook, err error) + + // CreatePlaybook stores a new playbook. + CreatePlaybook(ctx context.Context, playbook *cacao.Playbook) error + + // UpdatePlaybook replaces an existing playbook. + UpdatePlaybook(ctx context.Context, playbookID string, playbook *cacao.Playbook) error + + // DeletePlaybook removes a playbook. + DeletePlaybook(ctx context.Context, playbookID string) error + + // ListPlaybookMetas returns metadata for all playbooks (efficient list). + ListPlaybookMetas(ctx context.Context) (metas []playbooks.Meta, err error) +} + +// ============================================================================ +// DEPENDENCY NOTES +// ============================================================================ + +/* +Dependency Flow (what depends on what): + + runs.Runner (core kernel) + └─ owns: engine (decomposer factory), PlaybookStore, RunState + + FinRegistry (independent) + └─ owns: FinStore + + FinWorkService (leased work) + └─ owns: FinQueue, FinStore + + ManualInbox + └─ depends on: manual inbox + + PlaybookService (CRUD adapter) + └─ depends on: PlaybookStore + +HTTP Handlers (thin transport) + └─ depend on: these services + └─ do NOT depend on: stores, queues, runtime directly +*/ diff --git a/internal/services/playbook/service.go b/internal/services/playbook/service.go new file mode 100644 index 00000000..5acc7e1e --- /dev/null +++ b/internal/services/playbook/service.go @@ -0,0 +1,54 @@ +package playbook + +import ( + "context" + + "soarca/internal/playbooks" + "soarca/internal/storage" + "soarca/pkg/models/cacao" +) + +// Service implements the services.PlaybookService interface. +type Service struct { + store storage.PlaybookStore +} + +// New creates a new playbook service. +func New(store storage.PlaybookStore) *Service { + return &Service{store: store} +} + +// ListPlaybooks returns all stored playbooks. +func (s *Service) ListPlaybooks(ctx context.Context) ([]cacao.Playbook, error) { + return s.store.List(ctx) +} + +// GetPlaybook retrieves a playbook by ID. +func (s *Service) GetPlaybook(ctx context.Context, playbookID string) (*cacao.Playbook, error) { + playbook, err := s.store.Get(ctx, playbookID) + if err != nil { + return nil, err + } + return &playbook, nil +} + +// CreatePlaybook stores a new playbook. +func (s *Service) CreatePlaybook(ctx context.Context, playbook *cacao.Playbook) error { + return s.store.Create(ctx, *playbook) +} + +// UpdatePlaybook replaces an existing playbook. +func (s *Service) UpdatePlaybook(ctx context.Context, playbookID string, playbook *cacao.Playbook) error { + playbook.ID = playbookID + return s.store.Update(ctx, *playbook) +} + +// DeletePlaybook removes a playbook. +func (s *Service) DeletePlaybook(ctx context.Context, playbookID string) error { + return s.store.Delete(ctx, playbookID) +} + +// ListPlaybookMetas returns playbook metadata entries. +func (s *Service) ListPlaybookMetas(ctx context.Context) ([]playbooks.Meta, error) { + return s.store.ListMeta(ctx) +} diff --git a/internal/storage/playbooks.go b/internal/storage/playbooks.go new file mode 100644 index 00000000..d17467e4 --- /dev/null +++ b/internal/storage/playbooks.go @@ -0,0 +1,23 @@ +package storage + +import ( + "context" + + "soarca/internal/playbooks" + "soarca/pkg/models/cacao" +) + +type PlaybookStore interface { + // Create stores a new playbook. Returns ErrConflict if the ID already exists. + Create(ctx context.Context, pb cacao.Playbook) error + // Update replaces an existing playbook. Returns ErrNotFound if the ID does not exist. + Update(ctx context.Context, pb cacao.Playbook) error + // Get retrieves a playbook by ID. Returns ErrNotFound if it does not exist. + Get(ctx context.Context, id string) (cacao.Playbook, error) + // Delete removes a playbook by ID. Returns ErrNotFound if it does not exist. + Delete(ctx context.Context, id string) error + // List returns all stored playbooks. + List(ctx context.Context) ([]cacao.Playbook, error) + // ListMeta returns lightweight metadata for all stored playbooks. + ListMeta(ctx context.Context) ([]playbooks.Meta, error) +} diff --git a/internal/storage/sql/playbooks.go b/internal/storage/sql/playbooks.go new file mode 100644 index 00000000..d812b0f1 --- /dev/null +++ b/internal/storage/sql/playbooks.go @@ -0,0 +1,158 @@ +package sql + +import ( + "context" + databasesql "database/sql" + "encoding/json" + "errors" + "fmt" + + "soarca/internal/playbooks" + "soarca/internal/storage" + "soarca/pkg/models/cacao" +) + +type playbookStore struct { + db *databasesql.DB +} + +func (s *playbookStore) Create(ctx context.Context, pb cacao.Playbook) error { + doc, labels, err := encodePlaybook(pb) + if err != nil { + return err + } + + const query = `INSERT INTO playbooks + (id, name, description, created, modified, valid_from, valid_until, labels, doc) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)` + + _, err = s.db.ExecContext(ctx, query, + pb.ID, pb.Name, pb.Description, + pb.Created, pb.Modified, pb.ValidFrom, pb.ValidUntil, + labels, doc, + ) + if err != nil { + if isUniqueViolation(err) { + return storage.ErrConflict + } + return fmt.Errorf("create playbook: %w", err) + } + return nil +} + +func (s *playbookStore) Update(ctx context.Context, pb cacao.Playbook) error { + doc, labels, err := encodePlaybook(pb) + if err != nil { + return err + } + + const query = `UPDATE playbooks SET + name = $2, description = $3, created = $4, modified = $5, + valid_from = $6, valid_until = $7, labels = $8, doc = $9 + WHERE id = $1` + + result, err := s.db.ExecContext(ctx, query, + pb.ID, pb.Name, pb.Description, + pb.Created, pb.Modified, pb.ValidFrom, pb.ValidUntil, + labels, doc, + ) + if err != nil { + return fmt.Errorf("update playbook: %w", err) + } + return requireOneRow(result, "update playbook") +} + +func (s *playbookStore) Get(ctx context.Context, id string) (cacao.Playbook, error) { + var doc []byte + err := s.db.QueryRowContext(ctx, `SELECT doc FROM playbooks WHERE id = $1`, id).Scan(&doc) + if errors.Is(err, databasesql.ErrNoRows) { + return cacao.Playbook{}, storage.ErrNotFound + } + if err != nil { + return cacao.Playbook{}, fmt.Errorf("get playbook: %w", err) + } + + var pb cacao.Playbook + if err := json.Unmarshal(doc, &pb); err != nil { + return cacao.Playbook{}, fmt.Errorf("decode playbook %s: %w", id, err) + } + return pb, nil +} + +func (s *playbookStore) Delete(ctx context.Context, id string) error { + result, err := s.db.ExecContext(ctx, `DELETE FROM playbooks WHERE id = $1`, id) + if err != nil { + return fmt.Errorf("delete playbook: %w", err) + } + return requireOneRow(result, "delete playbook") +} + +func (s *playbookStore) List(ctx context.Context) ([]cacao.Playbook, error) { + rows, err := s.db.QueryContext(ctx, `SELECT doc FROM playbooks ORDER BY id`) + if err != nil { + return nil, fmt.Errorf("list playbooks: %w", err) + } + defer rows.Close() + + playbooks := make([]cacao.Playbook, 0) + for rows.Next() { + var doc []byte + if err := rows.Scan(&doc); err != nil { + return nil, fmt.Errorf("list playbooks: %w", err) + } + var pb cacao.Playbook + if err := json.Unmarshal(doc, &pb); err != nil { + return nil, fmt.Errorf("decode playbook: %w", err) + } + playbooks = append(playbooks, pb) + } + return playbooks, rows.Err() +} + +// ListMeta reads the extracted columns so whole playbooks never need decoding. +func (s *playbookStore) ListMeta(ctx context.Context) ([]playbooks.Meta, error) { + const query = `SELECT id, name, description, valid_from, valid_until, labels + FROM playbooks ORDER BY id` + + rows, err := s.db.QueryContext(ctx, query) + if err != nil { + return nil, fmt.Errorf("list playbook metadata: %w", err) + } + defer rows.Close() + + metas := make([]playbooks.Meta, 0) + for rows.Next() { + var ( + meta playbooks.Meta + validFrom databasesql.NullTime + validUntil databasesql.NullTime + labels []byte + ) + if err := rows.Scan(&meta.ID, &meta.Name, &meta.Description, &validFrom, &validUntil, &labels); err != nil { + return nil, fmt.Errorf("list playbook metadata: %w", err) + } + meta.ValidFrom = validFrom.Time + meta.ValidUntil = validUntil.Time + if err := json.Unmarshal(labels, &meta.Labels); err != nil { + return nil, fmt.Errorf("decode playbook labels: %w", err) + } + metas = append(metas, meta) + } + return metas, rows.Err() +} + +func encodePlaybook(pb cacao.Playbook) (doc []byte, labels []byte, err error) { + doc, err = json.Marshal(pb) + if err != nil { + return nil, nil, fmt.Errorf("encode playbook %s: %w", pb.ID, err) + } + labelValues := pb.Labels + if labelValues == nil { + labelValues = []string{} + } + labels, err = json.Marshal(labelValues) + if err != nil { + return nil, nil, fmt.Errorf("encode playbook labels %s: %w", pb.ID, err) + } + return doc, labels, nil +} diff --git a/internal/store/errors.go b/internal/store/errors.go new file mode 100644 index 00000000..20a8ceb3 --- /dev/null +++ b/internal/store/errors.go @@ -0,0 +1,8 @@ +package storage + +import "errors" + +var ( + ErrNotFound = errors.New("not found") + ErrConflict = errors.New("conflict") +) diff --git a/internal/store/fins.go b/internal/store/fins.go new file mode 100644 index 00000000..93000a47 --- /dev/null +++ b/internal/store/fins.go @@ -0,0 +1,23 @@ +package storage + +import ( + "context" + "time" + + "soarca/pkg/fins/protocol" +) + +type FinStore interface { + // Register persists a new Fin registration. Returns ErrConflict if the ID already exists. + Create(ctx context.Context, record fin.Record) error + // Get looks up a Fin by its FinId. Returns ErrNotFound if none exists. + Get(ctx context.Context, finID string) (fin.Record, error) + // GetByTokenHash looks up the Fin whose FinTokenHash matches. Returns ErrNotFound if none exists. + GetByTokenHash(ctx context.Context, tokenHash string) (fin.Record, error) + // List returns all registered Fins. + List(ctx context.Context) ([]fin.Record, error) + // Touch updates a Fin's LastSeen timestamp. Returns ErrNotFound if the Fin does not exist. + Touch(ctx context.Context, finID string, at time.Time) error + // Unregister removes a Fin registration. Returns ErrNotFound if none exists. + Delete(ctx context.Context, finID string) error +} diff --git a/internal/store/playbooks.go b/internal/store/playbooks.go new file mode 100644 index 00000000..e79f1d87 --- /dev/null +++ b/internal/store/playbooks.go @@ -0,0 +1,23 @@ +package storage + +import ( + "context" + + "soarca/internal/playbooks" + "soarca/pkg/cacao" +) + +type PlaybookStore interface { + // Create stores a new playbook. Returns ErrConflict if the ID already exists. + Create(ctx context.Context, pb cacao.Playbook) error + // Update replaces an existing playbook. Returns ErrNotFound if the ID does not exist. + Update(ctx context.Context, pb cacao.Playbook) error + // Get retrieves a playbook by ID. Returns ErrNotFound if it does not exist. + Get(ctx context.Context, id string) (cacao.Playbook, error) + // Delete removes a playbook by ID. Returns ErrNotFound if it does not exist. + Delete(ctx context.Context, id string) error + // List returns all stored playbooks. + List(ctx context.Context) ([]cacao.Playbook, error) + // ListMeta returns lightweight metadata for all stored playbooks. + ListMeta(ctx context.Context) ([]playbooks.Meta, error) +} diff --git a/internal/store/sql/fins.go b/internal/store/sql/fins.go new file mode 100644 index 00000000..39e41261 --- /dev/null +++ b/internal/store/sql/fins.go @@ -0,0 +1,131 @@ +package sql + +import ( + "context" + databasesql "database/sql" + "encoding/json" + "errors" + "fmt" + "time" + + "soarca/internal/store" + "soarca/pkg/fins/protocol" +) + +type finStore struct { + db *databasesql.DB +} + +func (s *finStore) Create(ctx context.Context, record fin.Record) error { + capabilities, err := encodeCapabilities(record) + if err != nil { + return err + } + + const query = `INSERT INTO fins + (fin_id, fin_token_hash, display_name, protocol_version, capabilities, registered_at, last_seen) + VALUES ($1, $2, $3, $4, $5, $6, $7)` + + _, err = s.db.ExecContext(ctx, query, + record.FinId, record.FinTokenHash, record.DisplayName, + record.ProtocolVersion, capabilities, + record.RegisteredAt, record.LastSeen, + ) + if err != nil { + if isUniqueViolation(err) { + return storage.ErrConflict + } + return fmt.Errorf("create fin: %w", err) + } + return nil +} + +func (s *finStore) Get(ctx context.Context, finID string) (fin.Record, error) { + return s.queryOne(ctx, finSelect+` WHERE fin_id = $1`, finID) +} + +func (s *finStore) GetByTokenHash(ctx context.Context, tokenHash string) (fin.Record, error) { + return s.queryOne(ctx, finSelect+` WHERE fin_token_hash = $1`, tokenHash) +} + +func (s *finStore) List(ctx context.Context) ([]fin.Record, error) { + rows, err := s.db.QueryContext(ctx, finSelect+` ORDER BY fin_id`) + if err != nil { + return nil, fmt.Errorf("list fins: %w", err) + } + defer rows.Close() + + records := make([]fin.Record, 0) + for rows.Next() { + record, err := scanFin(rows) + if err != nil { + return nil, err + } + records = append(records, record) + } + return records, rows.Err() +} + +func (s *finStore) Touch(ctx context.Context, finID string, at time.Time) error { + result, err := s.db.ExecContext(ctx, + `UPDATE fins SET last_seen = $2 WHERE fin_id = $1`, finID, at) + if err != nil { + return fmt.Errorf("touch fin: %w", err) + } + return requireOneRow(result, "touch fin") +} + +func (s *finStore) Delete(ctx context.Context, finID string) error { + result, err := s.db.ExecContext(ctx, `DELETE FROM fins WHERE fin_id = $1`, finID) + if err != nil { + return fmt.Errorf("delete fin: %w", err) + } + return requireOneRow(result, "delete fin") +} + +const finSelect = `SELECT fin_id, fin_token_hash, display_name, protocol_version, + capabilities, registered_at, last_seen FROM fins` + +func (s *finStore) queryOne(ctx context.Context, query string, arg any) (fin.Record, error) { + record, err := scanFin(s.db.QueryRowContext(ctx, query, arg)) + if errors.Is(err, databasesql.ErrNoRows) { + return fin.Record{}, storage.ErrNotFound + } + return record, err +} + +// scanner covers both *sql.Row and *sql.Rows. +type scanner interface { + Scan(dest ...any) error +} + +func scanFin(row scanner) (fin.Record, error) { + var ( + record fin.Record + capabilities []byte + ) + err := row.Scan( + &record.FinId, &record.FinTokenHash, &record.DisplayName, + &record.ProtocolVersion, &capabilities, + &record.RegisteredAt, &record.LastSeen, + ) + if err != nil { + return fin.Record{}, err + } + if err := json.Unmarshal(capabilities, &record.Capabilities); err != nil { + return fin.Record{}, fmt.Errorf("decode fin capabilities %s: %w", record.FinId, err) + } + return record, nil +} + +func encodeCapabilities(record fin.Record) ([]byte, error) { + capabilities := record.Capabilities + if capabilities == nil { + capabilities = []fin.Capability{} + } + encoded, err := json.Marshal(capabilities) + if err != nil { + return nil, fmt.Errorf("encode fin capabilities %s: %w", record.FinId, err) + } + return encoded, nil +} diff --git a/internal/store/sql/helpers.go b/internal/store/sql/helpers.go new file mode 100644 index 00000000..535bd9a2 --- /dev/null +++ b/internal/store/sql/helpers.go @@ -0,0 +1,34 @@ +package sql + +import ( + databasesql "database/sql" + "fmt" + "strings" + + "soarca/internal/store" +) + +// requireOneRow turns "statement affected nothing" into ErrNotFound, which is +// how the storage contract reports a missing row. +func requireOneRow(result databasesql.Result, action string) error { + affected, err := result.RowsAffected() + if err != nil { + return fmt.Errorf("%s: %w", action, err) + } + if affected == 0 { + return storage.ErrNotFound + } + return nil +} + +// isUniqueViolation reports whether err is a primary key or unique constraint +// failure. SQLite and PostgreSQL surface this differently and neither exposes a +// portable sentinel, so the check is on the message. +func isUniqueViolation(err error) bool { + if err == nil { + return false + } + msg := strings.ToLower(err.Error()) + return strings.Contains(msg, "unique constraint") || // sqlite + strings.Contains(msg, "duplicate key value") // postgres +} diff --git a/internal/store/sql/migrations/00001_init.sql b/internal/store/sql/migrations/00001_init.sql new file mode 100644 index 00000000..abedbb10 --- /dev/null +++ b/internal/store/sql/migrations/00001_init.sql @@ -0,0 +1,26 @@ +-- +goose Up +CREATE TABLE playbooks ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL DEFAULT '', + description TEXT NOT NULL DEFAULT '', + created TIMESTAMP, + modified TIMESTAMP, + valid_from TIMESTAMP, + valid_until TIMESTAMP, + labels TEXT NOT NULL DEFAULT '[]', + doc TEXT NOT NULL +); + +CREATE TABLE fins ( + fin_id TEXT PRIMARY KEY, + fin_token_hash TEXT NOT NULL UNIQUE, + display_name TEXT NOT NULL DEFAULT '', + protocol_version TEXT NOT NULL DEFAULT '', + capabilities TEXT NOT NULL DEFAULT '[]', + registered_at TIMESTAMP NOT NULL, + last_seen TIMESTAMP NOT NULL +); + +-- +goose Down +DROP TABLE fins; +DROP TABLE playbooks; diff --git a/internal/store/sql/playbooks.go b/internal/store/sql/playbooks.go new file mode 100644 index 00000000..8d89278f --- /dev/null +++ b/internal/store/sql/playbooks.go @@ -0,0 +1,158 @@ +package sql + +import ( + "context" + databasesql "database/sql" + "encoding/json" + "errors" + "fmt" + + "soarca/internal/store" + "soarca/internal/playbooks" + "soarca/pkg/cacao" +) + +type playbookStore struct { + db *databasesql.DB +} + +func (s *playbookStore) Create(ctx context.Context, pb cacao.Playbook) error { + doc, labels, err := encodePlaybook(pb) + if err != nil { + return err + } + + const query = `INSERT INTO playbooks + (id, name, description, created, modified, valid_from, valid_until, labels, doc) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)` + + _, err = s.db.ExecContext(ctx, query, + pb.ID, pb.Name, pb.Description, + pb.Created, pb.Modified, pb.ValidFrom, pb.ValidUntil, + labels, doc, + ) + if err != nil { + if isUniqueViolation(err) { + return storage.ErrConflict + } + return fmt.Errorf("create playbook: %w", err) + } + return nil +} + +func (s *playbookStore) Update(ctx context.Context, pb cacao.Playbook) error { + doc, labels, err := encodePlaybook(pb) + if err != nil { + return err + } + + const query = `UPDATE playbooks SET + name = $2, description = $3, created = $4, modified = $5, + valid_from = $6, valid_until = $7, labels = $8, doc = $9 + WHERE id = $1` + + result, err := s.db.ExecContext(ctx, query, + pb.ID, pb.Name, pb.Description, + pb.Created, pb.Modified, pb.ValidFrom, pb.ValidUntil, + labels, doc, + ) + if err != nil { + return fmt.Errorf("update playbook: %w", err) + } + return requireOneRow(result, "update playbook") +} + +func (s *playbookStore) Get(ctx context.Context, id string) (cacao.Playbook, error) { + var doc []byte + err := s.db.QueryRowContext(ctx, `SELECT doc FROM playbooks WHERE id = $1`, id).Scan(&doc) + if errors.Is(err, databasesql.ErrNoRows) { + return cacao.Playbook{}, storage.ErrNotFound + } + if err != nil { + return cacao.Playbook{}, fmt.Errorf("get playbook: %w", err) + } + + var pb cacao.Playbook + if err := json.Unmarshal(doc, &pb); err != nil { + return cacao.Playbook{}, fmt.Errorf("decode playbook %s: %w", id, err) + } + return pb, nil +} + +func (s *playbookStore) Delete(ctx context.Context, id string) error { + result, err := s.db.ExecContext(ctx, `DELETE FROM playbooks WHERE id = $1`, id) + if err != nil { + return fmt.Errorf("delete playbook: %w", err) + } + return requireOneRow(result, "delete playbook") +} + +func (s *playbookStore) List(ctx context.Context) ([]cacao.Playbook, error) { + rows, err := s.db.QueryContext(ctx, `SELECT doc FROM playbooks ORDER BY id`) + if err != nil { + return nil, fmt.Errorf("list playbooks: %w", err) + } + defer rows.Close() + + playbooks := make([]cacao.Playbook, 0) + for rows.Next() { + var doc []byte + if err := rows.Scan(&doc); err != nil { + return nil, fmt.Errorf("list playbooks: %w", err) + } + var pb cacao.Playbook + if err := json.Unmarshal(doc, &pb); err != nil { + return nil, fmt.Errorf("decode playbook: %w", err) + } + playbooks = append(playbooks, pb) + } + return playbooks, rows.Err() +} + +// ListMeta reads the extracted columns so whole playbooks never need decoding. +func (s *playbookStore) ListMeta(ctx context.Context) ([]playbooks.Meta, error) { + const query = `SELECT id, name, description, valid_from, valid_until, labels + FROM playbooks ORDER BY id` + + rows, err := s.db.QueryContext(ctx, query) + if err != nil { + return nil, fmt.Errorf("list playbook metadata: %w", err) + } + defer rows.Close() + + metas := make([]playbooks.Meta, 0) + for rows.Next() { + var ( + meta playbooks.Meta + validFrom databasesql.NullTime + validUntil databasesql.NullTime + labels []byte + ) + if err := rows.Scan(&meta.ID, &meta.Name, &meta.Description, &validFrom, &validUntil, &labels); err != nil { + return nil, fmt.Errorf("list playbook metadata: %w", err) + } + meta.ValidFrom = validFrom.Time + meta.ValidUntil = validUntil.Time + if err := json.Unmarshal(labels, &meta.Labels); err != nil { + return nil, fmt.Errorf("decode playbook labels: %w", err) + } + metas = append(metas, meta) + } + return metas, rows.Err() +} + +func encodePlaybook(pb cacao.Playbook) (doc []byte, labels []byte, err error) { + doc, err = json.Marshal(pb) + if err != nil { + return nil, nil, fmt.Errorf("encode playbook %s: %w", pb.ID, err) + } + labelValues := pb.Labels + if labelValues == nil { + labelValues = []string{} + } + labels, err = json.Marshal(labelValues) + if err != nil { + return nil, nil, fmt.Errorf("encode playbook labels %s: %w", pb.ID, err) + } + return doc, labels, nil +} diff --git a/internal/store/sql/sql_test.go b/internal/store/sql/sql_test.go new file mode 100644 index 00000000..7d2c91e3 --- /dev/null +++ b/internal/store/sql/sql_test.go @@ -0,0 +1,442 @@ +package sql + +import ( + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "soarca/internal/store" + "soarca/pkg/cacao" + "soarca/pkg/fins/protocol" +) + +// newTestStore gives each test its own private in-memory database. +func newTestStore(t *testing.T) *Store { + t.Helper() + + store, err := New(context.Background(), "file:"+t.Name()+"?mode=memory&cache=shared") + if err != nil { + t.Fatalf("New() returned error: %v", err) + } + t.Cleanup(func() { _ = store.Close(context.Background()) }) + return store +} + +func testPlaybook(id string) cacao.Playbook { + return cacao.Playbook{ + ID: id, + Name: "Test Playbook", + Description: "a playbook", + ValidFrom: time.Date(2024, 1, 1, 9, 0, 0, 0, time.UTC), + ValidUntil: time.Date(2124, 1, 1, 9, 0, 0, 0, time.UTC), + Labels: []string{"soarca", "test"}, + Workflow: cacao.Workflow{ + "start--test": cacao.Step{ID: "start--test", Type: cacao.StepTypeStart}, + }, + } +} + +func TestPlaybookCreateAndGet(t *testing.T) { + store := newTestStore(t) + ctx := context.Background() + pb := testPlaybook("playbook--1") + + if err := store.Playbooks().Create(ctx, pb); err != nil { + t.Fatalf("Create() returned error: %v", err) + } + + got, err := store.Playbooks().Get(ctx, pb.ID) + if err != nil { + t.Fatalf("Get() returned error: %v", err) + } + if got.ID != pb.ID || got.Name != pb.Name { + t.Errorf("Get() = %s/%s, want %s/%s", got.ID, got.Name, pb.ID, pb.Name) + } + // The whole document round-trips, not just the extracted columns. + if len(got.Workflow) != 1 { + t.Errorf("Get() workflow has %d steps, want 1", len(got.Workflow)) + } +} + +func TestPlaybookCreateDuplicateIsConflict(t *testing.T) { + store := newTestStore(t) + ctx := context.Background() + pb := testPlaybook("playbook--dup") + + if err := store.Playbooks().Create(ctx, pb); err != nil { + t.Fatalf("Create() returned error: %v", err) + } + err := store.Playbooks().Create(ctx, pb) + if !errors.Is(err, storage.ErrConflict) { + t.Errorf("Create() duplicate = %v, want ErrConflict", err) + } +} + +func TestPlaybookGetMissingIsNotFound(t *testing.T) { + store := newTestStore(t) + + _, err := store.Playbooks().Get(context.Background(), "playbook--missing") + if !errors.Is(err, storage.ErrNotFound) { + t.Errorf("Get() missing = %v, want ErrNotFound", err) + } +} + +func TestPlaybookUpdate(t *testing.T) { + store := newTestStore(t) + ctx := context.Background() + pb := testPlaybook("playbook--upd") + + if err := store.Playbooks().Create(ctx, pb); err != nil { + t.Fatalf("Create() returned error: %v", err) + } + + pb.Name = "Renamed" + if err := store.Playbooks().Update(ctx, pb); err != nil { + t.Fatalf("Update() returned error: %v", err) + } + + got, err := store.Playbooks().Get(ctx, pb.ID) + if err != nil { + t.Fatalf("Get() returned error: %v", err) + } + if got.Name != "Renamed" { + t.Errorf("Get() name = %q, want %q", got.Name, "Renamed") + } +} + +func TestPlaybookUpdateMissingIsNotFound(t *testing.T) { + store := newTestStore(t) + + err := store.Playbooks().Update(context.Background(), testPlaybook("playbook--nope")) + if !errors.Is(err, storage.ErrNotFound) { + t.Errorf("Update() missing = %v, want ErrNotFound", err) + } +} + +func TestPlaybookDelete(t *testing.T) { + store := newTestStore(t) + ctx := context.Background() + pb := testPlaybook("playbook--del") + + if err := store.Playbooks().Create(ctx, pb); err != nil { + t.Fatalf("Create() returned error: %v", err) + } + if err := store.Playbooks().Delete(ctx, pb.ID); err != nil { + t.Fatalf("Delete() returned error: %v", err) + } + if _, err := store.Playbooks().Get(ctx, pb.ID); !errors.Is(err, storage.ErrNotFound) { + t.Errorf("Get() after delete = %v, want ErrNotFound", err) + } + if err := store.Playbooks().Delete(ctx, pb.ID); !errors.Is(err, storage.ErrNotFound) { + t.Errorf("Delete() missing = %v, want ErrNotFound", err) + } +} + +func TestPlaybookListAndListMeta(t *testing.T) { + store := newTestStore(t) + ctx := context.Background() + + for _, id := range []string{"playbook--a", "playbook--b"} { + if err := store.Playbooks().Create(ctx, testPlaybook(id)); err != nil { + t.Fatalf("Create(%s) returned error: %v", id, err) + } + } + + list, err := store.Playbooks().List(ctx) + if err != nil { + t.Fatalf("List() returned error: %v", err) + } + if len(list) != 2 { + t.Fatalf("List() returned %d playbooks, want 2", len(list)) + } + + metas, err := store.Playbooks().ListMeta(ctx) + if err != nil { + t.Fatalf("ListMeta() returned error: %v", err) + } + if len(metas) != 2 { + t.Fatalf("ListMeta() returned %d metas, want 2", len(metas)) + } + if metas[0].ID != "playbook--a" { + t.Errorf("ListMeta()[0].ID = %q, want playbook--a", metas[0].ID) + } + if len(metas[0].Labels) != 2 { + t.Errorf("ListMeta()[0].Labels = %v, want 2 labels", metas[0].Labels) + } + if metas[0].ValidFrom.IsZero() { + t.Error("ListMeta()[0].ValidFrom is zero, want the stored timestamp") + } +} + +func TestPlaybookListIsEmptyNotNil(t *testing.T) { + store := newTestStore(t) + + list, err := store.Playbooks().List(context.Background()) + if err != nil { + t.Fatalf("List() returned error: %v", err) + } + if list == nil { + t.Error("List() returned nil, want empty slice") + } +} + +func testFin(id, tokenHash string) fin.Record { + return fin.Record{ + FinId: id, + FinTokenHash: tokenHash, + DisplayName: "test-fin", + ProtocolVersion: "1", + Capabilities: []fin.Capability{{Type: "soarca-fin-test"}}, + RegisteredAt: time.Date(2024, 1, 1, 9, 0, 0, 0, time.UTC), + LastSeen: time.Date(2024, 1, 1, 9, 0, 0, 0, time.UTC), + } +} + +func TestFinCreateAndGet(t *testing.T) { + store := newTestStore(t) + ctx := context.Background() + record := testFin("fin--1", "hash-1") + + if err := store.Fins().Create(ctx, record); err != nil { + t.Fatalf("Create() returned error: %v", err) + } + + got, err := store.Fins().Get(ctx, record.FinId) + if err != nil { + t.Fatalf("Get() returned error: %v", err) + } + if got.DisplayName != record.DisplayName { + t.Errorf("Get() display name = %q, want %q", got.DisplayName, record.DisplayName) + } + if len(got.Capabilities) != 1 || got.Capabilities[0].Type != "soarca-fin-test" { + t.Errorf("Get() capabilities = %v, want one soarca-fin-test", got.Capabilities) + } +} + +func TestFinGetByTokenHash(t *testing.T) { + store := newTestStore(t) + ctx := context.Background() + record := testFin("fin--token", "hash-token") + + if err := store.Fins().Create(ctx, record); err != nil { + t.Fatalf("Create() returned error: %v", err) + } + + got, err := store.Fins().GetByTokenHash(ctx, "hash-token") + if err != nil { + t.Fatalf("GetByTokenHash() returned error: %v", err) + } + if got.FinId != record.FinId { + t.Errorf("GetByTokenHash() fin id = %q, want %q", got.FinId, record.FinId) + } + + if _, err := store.Fins().GetByTokenHash(ctx, "nope"); !errors.Is(err, storage.ErrNotFound) { + t.Errorf("GetByTokenHash() unknown = %v, want ErrNotFound", err) + } +} + +func TestFinDuplicateTokenHashIsConflict(t *testing.T) { + store := newTestStore(t) + ctx := context.Background() + + if err := store.Fins().Create(ctx, testFin("fin--a", "shared-hash")); err != nil { + t.Fatalf("Create() returned error: %v", err) + } + err := store.Fins().Create(ctx, testFin("fin--b", "shared-hash")) + if !errors.Is(err, storage.ErrConflict) { + t.Errorf("Create() duplicate token hash = %v, want ErrConflict", err) + } +} + +func TestFinTouch(t *testing.T) { + store := newTestStore(t) + ctx := context.Background() + record := testFin("fin--touch", "hash-touch") + + if err := store.Fins().Create(ctx, record); err != nil { + t.Fatalf("Create() returned error: %v", err) + } + + later := record.LastSeen.Add(time.Hour) + if err := store.Fins().Touch(ctx, record.FinId, later); err != nil { + t.Fatalf("Touch() returned error: %v", err) + } + + got, err := store.Fins().Get(ctx, record.FinId) + if err != nil { + t.Fatalf("Get() returned error: %v", err) + } + if !got.LastSeen.UTC().Equal(later) { + t.Errorf("LastSeen = %v, want %v", got.LastSeen.UTC(), later) + } + + if err := store.Fins().Touch(ctx, "fin--missing", later); !errors.Is(err, storage.ErrNotFound) { + t.Errorf("Touch() missing = %v, want ErrNotFound", err) + } +} + +func TestFinListAndDelete(t *testing.T) { + store := newTestStore(t) + ctx := context.Background() + + if err := store.Fins().Create(ctx, testFin("fin--1", "h1")); err != nil { + t.Fatalf("Create() returned error: %v", err) + } + if err := store.Fins().Create(ctx, testFin("fin--2", "h2")); err != nil { + t.Fatalf("Create() returned error: %v", err) + } + + records, err := store.Fins().List(ctx) + if err != nil { + t.Fatalf("List() returned error: %v", err) + } + if len(records) != 2 { + t.Fatalf("List() returned %d fins, want 2", len(records)) + } + + if err := store.Fins().Delete(ctx, "fin--1"); err != nil { + t.Fatalf("Delete() returned error: %v", err) + } + if _, err := store.Fins().Get(ctx, "fin--1"); !errors.Is(err, storage.ErrNotFound) { + t.Errorf("Get() after delete = %v, want ErrNotFound", err) + } + if err := store.Fins().Delete(ctx, "fin--1"); !errors.Is(err, storage.ErrNotFound) { + t.Errorf("Delete() missing = %v, want ErrNotFound", err) + } +} + +func TestMigrationsAreIdempotent(t *testing.T) { + ctx := context.Background() + url := "file:migrations-idempotent?mode=memory&cache=shared" + + first, err := New(ctx, url) + if err != nil { + t.Fatalf("first New() returned error: %v", err) + } + defer first.Close(ctx) + + second, err := New(ctx, url) + if err != nil { + t.Fatalf("second New() returned error: %v", err) + } + defer second.Close(ctx) +} + +func TestParseURL(t *testing.T) { + const fileDefaults = "?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)" + const memoryDefaults = "?cache=shared&_pragma=busy_timeout(5000)" + + tests := []struct { + url string + wantDriver string + wantDSN string + wantPath string + }{ + {"sqlite://soarca.db", "sqlite", "file:soarca.db" + fileDefaults, "soarca.db"}, + {"sqlite://data/soarca.db", "sqlite", "file:data/soarca.db" + fileDefaults, "data/soarca.db"}, + {"sqlite:///var/lib/soarca.db", "sqlite", "file:/var/lib/soarca.db" + fileDefaults, "/var/lib/soarca.db"}, + {"sqlite3://soarca.db", "sqlite", "file:soarca.db" + fileDefaults, "soarca.db"}, + {"sqlite://:memory:", "sqlite", "file::memory:" + memoryDefaults, ""}, + {"./soarca.db", "sqlite", "file:./soarca.db" + fileDefaults, "./soarca.db"}, + // An explicit query takes over completely. + {"sqlite://soarca.db?_pragma=journal_mode(DELETE)", "sqlite", "file:soarca.db?_pragma=journal_mode(DELETE)", "soarca.db"}, + // A raw file: DSN is never rewritten. + {"file:test?mode=memory", "sqlite", "file:test?mode=memory", ""}, + {"postgres://u:p@host:5432/soarca", "pgx", "postgres://u:p@host:5432/soarca", ""}, + {"postgresql://u:p@host:5432/soarca", "pgx", "postgresql://u:p@host:5432/soarca", ""}, + } + + for _, test := range tests { + got, err := parseURL(test.url) + if err != nil { + t.Errorf("parseURL(%q) returned error: %v", test.url, err) + continue + } + if got.driverName != test.wantDriver || got.dsn != test.wantDSN || got.filePath != test.wantPath { + t.Errorf("parseURL(%q) = %q/%q/%q, want %q/%q/%q", + test.url, got.driverName, got.dsn, got.filePath, + test.wantDriver, test.wantDSN, test.wantPath) + } + } +} + +func TestParseURLRejectsUnknownScheme(t *testing.T) { + if _, err := parseURL("mongodb://localhost:27017"); err == nil { + t.Error("parseURL() accepted a mongodb URL, want an error") + } + if _, err := parseURL(""); err == nil { + t.Error("parseURL() accepted an empty URL, want an error") + } +} + +func TestSQLiteFileCreatesParentDirectory(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "nested", "dir", "soarca.db") + + store, err := New(ctx, "sqlite://"+path) + if err != nil { + t.Fatalf("New() returned error: %v", err) + } + defer store.Close(ctx) + + if _, err := os.Stat(path); err != nil { + t.Errorf("database file was not created: %v", err) + } +} + +// The pool is left at its defaults, so an in-memory database must survive +// queries landing on different connections. +func TestInMemoryStoreWorksAcrossPooledConnections(t *testing.T) { + ctx := context.Background() + + store, err := New(ctx, "sqlite://:memory:") + if err != nil { + t.Fatalf("New() returned error: %v", err) + } + defer store.Close(ctx) + + if err := store.Playbooks().Create(ctx, testPlaybook("playbook--pooled")); err != nil { + t.Fatalf("Create() returned error: %v", err) + } + + // Hold one connection open so the reads below need a second one. + tx, err := store.db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("BeginTx() returned error: %v", err) + } + defer tx.Rollback() + + for i := 0; i < 5; i++ { + if _, err := store.Playbooks().Get(ctx, "playbook--pooled"); err != nil { + t.Fatalf("Get() on pooled connection returned error: %v", err) + } + } + + if open := store.db.Stats().OpenConnections; open < 2 { + t.Errorf("only %d connection(s) opened, test did not exercise pooling", open) + } +} + +// A file database should end up in WAL mode so readers do not block a writer. +func TestFileStoreUsesWAL(t *testing.T) { + ctx := context.Background() + path := filepath.Join(t.TempDir(), "soarca.db") + + store, err := New(ctx, "sqlite://"+path) + if err != nil { + t.Fatalf("New() returned error: %v", err) + } + defer store.Close(ctx) + + var mode string + if err := store.db.QueryRowContext(ctx, "PRAGMA journal_mode").Scan(&mode); err != nil { + t.Fatalf("PRAGMA journal_mode returned error: %v", err) + } + if !strings.EqualFold(mode, "wal") { + t.Errorf("journal_mode = %q, want wal", mode) + } +} diff --git a/internal/store/sql/store.go b/internal/store/sql/store.go new file mode 100644 index 00000000..8a77dc14 --- /dev/null +++ b/internal/store/sql/store.go @@ -0,0 +1,190 @@ +// Package sql implements the storage interfaces on SQLite and PostgreSQL. +// +// Documents (playbooks, fin capabilities) are stored as JSON text; columns are +// extracted only where they need to be queried or listed. +package sql + +import ( + "context" + databasesql "database/sql" + "embed" + "fmt" + "os" + "path/filepath" + "strings" + + "soarca/internal/store" + + "github.com/pressly/goose/v3" + + _ "github.com/jackc/pgx/v5/stdlib" + _ "modernc.org/sqlite" +) + +//go:embed migrations/*.sql +var migrations embed.FS + +// target is everything needed to open and migrate one database. +type target struct { + driverName string // as registered with database/sql + dialect string // as known to goose + dsn string + filePath string // SQLite file to create a directory for, "" otherwise +} + +type Store struct { + db *databasesql.DB + playbooks *playbookStore + fins *finStore +} + +// New opens the database named by a URL, applies migrations, and returns a +// ready store. The scheme selects the backend: +// +// sqlite://soarca.db +// sqlite://:memory: +// file:name?mode=memory&cache=shared +// postgres://user:pass@host:5432/soarca?sslmode=disable +func New(ctx context.Context, databaseURL string) (*Store, error) { + target, err := parseURL(databaseURL) + if err != nil { + return nil, err + } + + if err := ensureParentDir(target.filePath); err != nil { + return nil, err + } + + db, err := databasesql.Open(target.driverName, target.dsn) + if err != nil { + return nil, fmt.Errorf("open %s database: %w", target.driverName, err) + } + + if err := db.PingContext(ctx); err != nil { + db.Close() + return nil, fmt.Errorf("connect to %s database: %w", target.driverName, err) + } + + if err := migrate(db, target.dialect); err != nil { + db.Close() + return nil, err + } + + return &Store{ + db: db, + playbooks: &playbookStore{db: db}, + fins: &finStore{db: db}, + }, nil +} + +// parseURL picks the driver from the URL scheme. database/sql has no notion of +// schemes: it takes a registered driver name and a driver-specific DSN, so the +// mapping has to happen here. A bare path is treated as SQLite so +// DATABASE_URL=./soarca.db works. +func parseURL(raw string) (target, error) { + trimmed := strings.TrimSpace(raw) + + sqlite := func(path string) (target, error) { + return target{ + driverName: "sqlite", + dialect: "sqlite3", + dsn: sqliteDSN(path), + filePath: sqliteFilePath(path), + }, nil + } + + switch { + case trimmed == "": + return target{}, fmt.Errorf("database URL is empty") + case strings.HasPrefix(trimmed, "postgres://"), strings.HasPrefix(trimmed, "postgresql://"): + return target{driverName: "pgx", dialect: "postgres", dsn: trimmed}, nil + case strings.HasPrefix(trimmed, "sqlite://"): + return sqlite(strings.TrimPrefix(trimmed, "sqlite://")) + case strings.HasPrefix(trimmed, "sqlite3://"): + return sqlite(strings.TrimPrefix(trimmed, "sqlite3://")) + case strings.HasPrefix(trimmed, "file:"): + // Raw driver DSN: the caller owns every parameter. + return target{driverName: "sqlite", dialect: "sqlite3", dsn: trimmed}, nil + case !strings.Contains(trimmed, "://"): + return sqlite(trimmed) + default: + scheme, _, _ := strings.Cut(trimmed, "://") + return target{}, fmt.Errorf("unsupported database URL scheme %q, want sqlite or postgres", scheme) + } +} + +const ( + // busyTimeout stops concurrent writers failing immediately with + // SQLITE_BUSY; they wait for the lock instead. + busyTimeout = "_pragma=busy_timeout(5000)" + // walJournal lets readers run alongside a writer on a file database. + walJournal = "_pragma=journal_mode(WAL)" +) + +// sqliteDSN turns a path from a sqlite:// URL into a driver DSN, applying +// defaults that make the pool behave normally. Supplying any query string +// takes full control and disables these defaults. +func sqliteDSN(raw string) string { + path, query, hasQuery := strings.Cut(raw, "?") + + if path == "" || path == ":memory:" { + // Without a shared cache each pooled connection would get its own + // empty database, so the migrated schema would keep disappearing. + if !hasQuery { + query = "cache=shared&" + busyTimeout + } + return "file::memory:?" + query + } + + if !hasQuery { + query = walJournal + "&" + busyTimeout + } + return "file:" + path + "?" + query +} + +// sqliteFilePath returns the file a sqlite:// URL points at, or "" for an +// in-memory database. +func sqliteFilePath(raw string) string { + path, _, _ := strings.Cut(raw, "?") + if path == "" || path == ":memory:" { + return "" + } + return path +} + +// ensureParentDir creates the directory for a SQLite file so a configured path +// like sqlite://data/soarca.db works without manual setup. +func ensureParentDir(path string) error { + if path == "" { + return nil + } + dir := filepath.Dir(path) + if dir == "." || dir == "" { + return nil + } + if err := os.MkdirAll(dir, 0o750); err != nil { + return fmt.Errorf("create database directory %s: %w", dir, err) + } + return nil +} + +func migrate(db *databasesql.DB, dialect string) error { + goose.SetBaseFS(migrations) + goose.SetLogger(goose.NopLogger()) + if err := goose.SetDialect(dialect); err != nil { + return fmt.Errorf("set migration dialect: %w", err) + } + if err := goose.Up(db, "migrations"); err != nil { + return fmt.Errorf("apply migrations: %w", err) + } + return nil +} + +func (s *Store) Playbooks() storage.PlaybookStore { return s.playbooks } + +func (s *Store) Fins() storage.FinStore { return s.fins } + +func (s *Store) Close(ctx context.Context) error { + _ = ctx + return s.db.Close() +} diff --git a/internal/store/storagetest/store.go b/internal/store/storagetest/store.go new file mode 100644 index 00000000..d96a369a --- /dev/null +++ b/internal/store/storagetest/store.go @@ -0,0 +1,24 @@ +// Package storagetest provides a disposable store for tests. +package storagetest + +import ( + "context" + "testing" + + "soarca/internal/store" + storagesql "soarca/internal/store/sql" +) + +// New returns a store backed by a SQLite database private to this test, so +// tests exercise the same SQL path as production. +func New(t *testing.T) storage.Store { + t.Helper() + + store, err := storagesql.New(context.Background(), + "file:"+t.Name()+"?mode=memory&cache=shared") + if err != nil { + t.Fatalf("storagetest.New() returned error: %v", err) + } + t.Cleanup(func() { _ = store.Close(context.Background()) }) + return store +} diff --git a/internal/store/store.go b/internal/store/store.go new file mode 100644 index 00000000..d9a9d35b --- /dev/null +++ b/internal/store/store.go @@ -0,0 +1,9 @@ +package storage + +import "context" + +type Store interface { + Playbooks() PlaybookStore + Fins() FinStore + Close(ctx context.Context) error +} diff --git a/internal/transport/http/handlers/api_test.go b/internal/transport/http/handlers/api_test.go new file mode 100644 index 00000000..bac90ade --- /dev/null +++ b/internal/transport/http/handlers/api_test.go @@ -0,0 +1,111 @@ +package api + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + + finservice "soarca/internal/fins" + "soarca/internal/store/storagetest" + "soarca/internal/transport/http/handlers/fin" + "soarca/internal/workflow/capability/fin/queue" + finmodels "soarca/pkg/fins/protocol" + "soarca/test/unittest/mocks/mock_guid" + + "github.com/gin-gonic/gin" + "github.com/go-playground/assert/v2" + "github.com/google/uuid" +) + +// requireAdminHeader is a stand-in for gauth's real JWT middleware +// (internal/controller/controller.go's intializeAuthenticationMiddleware) - +// it only exists to exercise gin's actual registration-order semantics +// without needing a real OIDC/JWKS setup in a unit test. It rejects any +// request that doesn't carry X-Admin-Test: yes. +func requireAdminHeader(g *gin.Context) { + if g.GetHeader("X-Admin-Test") != "yes" { + g.AbortWithStatus(http.StatusUnauthorized) + return + } + g.Next() +} + +// This test pins the exact ordering hazard the Fin routes must avoid: gin's +// engine.Use() only affects routes registered *after* it is called - routes +// already registered keep whatever handler chain they had at registration +// time. FinPublicRoutes must therefore be registered before any global +// admin-auth middleware is installed (see FinPublic's doc comment and +// internal/controller/controller.go's ordering), while FinAdminRoutes is +// expected to sit behind it, like the rest of the admin API. +func TestFinPublicRoutesAreExemptFromAdminAuthButFinAdminRoutesAreNot(t *testing.T) { + gin.SetMode(gin.TestMode) + app := gin.New() + + repository := storagetest.New(t).Fins() + jobQueue := queue.New() + t.Cleanup(jobQueue.Close) + guidMock := new(mock_guid.Mock_Guid) + guidMock.On("New").Return(uuid.New()) + + finHandler := fin.NewFinHandler( + finservice.NewRegistry(repository, finservice.RegistryConfig{ + RegistrationToken: "test-registration-token", + }, guidMock), + finservice.NewWorkService(repository, jobQueue, finservice.WorkServiceConfig{ + LongPollTimeoutSeconds: 1, + JobLeaseSeconds: 60, + }), + fin.Config{ + RegistrationToken: "test-registration-token", + PollIntervalSeconds: 5, + LongPollTimeoutSeconds: 1, + JobLeaseSeconds: 60, + }, + ) + + // Simulates: routes.FinPublic(app, finHandler) called before + // intializeAuthenticationMiddleware(app) in controller.go. + FinPublicRoutes(app, finHandler) + + // Simulates: intializeAuthenticationMiddleware(app) installing the + // global admin-auth middleware. + app.Use(requireAdminHeader) + + // Simulates: routes.FinAdmin(app, finHandler) called afterwards, + // alongside the other admin routes (routes.Api, routes.Manual, etc.). + FinAdminRoutes(app, finHandler) + + registerRequest := httptest.NewRequest(http.MethodPost, "/fin/register", jsonBody(t, finmodels.RegisterRequest{ + RegistrationToken: "test-registration-token", + Capabilities: []finmodels.Capability{{Type: "pong"}}, + })) + registerRequest.Header.Set("Content-Type", "application/json") + registerRecorder := httptest.NewRecorder() + app.ServeHTTP(registerRecorder, registerRequest) + assert.Equal(t, registerRecorder.Code, http.StatusCreated) + + listRequest := httptest.NewRequest(http.MethodGet, "/fin/", nil) + listRecorder := httptest.NewRecorder() + app.ServeHTTP(listRecorder, listRequest) + if listRecorder.Code != http.StatusUnauthorized { + t.Fatalf("expected GET /fin/ to require admin auth, got %d", listRecorder.Code) + } + + listRequest = httptest.NewRequest(http.MethodGet, "/fin/", nil) + listRequest.Header.Set("X-Admin-Test", "yes") + listRecorder = httptest.NewRecorder() + app.ServeHTTP(listRecorder, listRequest) + assert.Equal(t, listRecorder.Code, http.StatusOK) +} + +func jsonBody(t *testing.T, v any) io.Reader { + t.Helper() + data, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + return bytes.NewReader(data) +} diff --git a/pkg/api/error/error.go b/internal/transport/http/handlers/error/response.go similarity index 100% rename from pkg/api/error/error.go rename to internal/transport/http/handlers/error/response.go diff --git a/internal/transport/http/handlers/fin/fin_api.go b/internal/transport/http/handlers/fin/fin_api.go new file mode 100644 index 00000000..4eaa86ce --- /dev/null +++ b/internal/transport/http/handlers/fin/fin_api.go @@ -0,0 +1,300 @@ +package fin + +import ( + "errors" + "net/http" + "reflect" + "strings" + "time" + + "soarca/internal/logger" + "soarca/internal/orchestrator" + apiError "soarca/internal/transport/http/handlers/error" + "soarca/pkg/fins/protocol" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" +) + +type Empty struct{} + +var log *logger.Log + +func init() { + log = logger.Logger(reflect.TypeOf(Empty{}).PkgPath(), logger.Info, "", logger.Json) +} + +const finContextKey = "fin_record" + +type Config struct { + RegistrationToken string + PollIntervalSeconds int + LongPollTimeoutSeconds int + JobLeaseSeconds int + StaleAfter time.Duration +} + +// FinHandler is the HTTP adapter for FIN operations. +type FinHandler struct { + registry orchestrator.FinRegistry + workService orchestrator.FinWorkService + config Config +} + +// NewFinHandler creates a new FIN HTTP handler with service dependencies. +func NewFinHandler(registry orchestrator.FinRegistry, workService orchestrator.FinWorkService, config Config) *FinHandler { + return &FinHandler{ + registry: registry, + workService: workService, + config: config, + } +} + +func (h *FinHandler) Register(g *gin.Context) { + const route = "POST /fin/register" + + var request fin.RegisterRequest + if err := g.ShouldBindJSON(&request); err != nil { + log.Error(err) + apiError.SendErrorResponse(g, http.StatusBadRequest, "Failed to parse registration request", route, err.Error()) + return + } + + finID, finToken, err := h.registry.RegisterFin(g.Request.Context(), request) + if err != nil { + h.sendRegistrationError(g, route, err) + return + } + + g.JSON(http.StatusCreated, fin.RegisterResponse{ + FinId: finID, + FinToken: finToken, + PollIntervalSeconds: h.config.PollIntervalSeconds, + LongPollTimeoutSeconds: h.config.LongPollTimeoutSeconds, + JobLeaseSeconds: h.config.JobLeaseSeconds, + }) +} + +func (h *FinHandler) RequireFinToken(g *gin.Context) { + const route = "fin bearer auth" + presentedToken, ok := bearerToken(g) + if !ok { + apiError.SendErrorResponse(g, http.StatusUnauthorized, "Missing or malformed Authorization header", route, "") + g.Abort() + return + } + + // Validate that the token is registered + if _, err := h.registry.ValidateToken(g.Request.Context(), presentedToken); err != nil { + apiError.SendErrorResponse(g, http.StatusUnauthorized, "Invalid or unknown fin token", route, "") + g.Abort() + return + } + + // Store the token in context for handlers to use + g.Set(finContextKey, presentedToken) + g.Next() +} + +func (h *FinHandler) Poll(g *gin.Context) { + finToken, ok := h.getFinToken(g) + if !ok { + apiError.SendErrorResponse(g, http.StatusUnauthorized, "FIN token not found in context", "POST /fin/poll", "") + return + } + + var request fin.PollRequest + if g.Request.ContentLength > 0 { + if err := g.ShouldBindJSON(&request); err != nil { + log.Error(err) + apiError.SendErrorResponse(g, http.StatusBadRequest, "Failed to parse poll request", "POST /fin/poll", err.Error()) + return + } + } + + job, err := h.workService.PollJob(g.Request.Context(), finToken, request) + if err != nil { + // Poll timeout or context cancellation -> return no content + g.Status(http.StatusNoContent) + return + } + + g.JSON(http.StatusOK, fin.PollResponse{Job: *job}) +} + +func (h *FinHandler) SubmitResult(g *gin.Context) { + finToken, ok := h.getFinToken(g) + if !ok { + apiError.SendErrorResponse(g, http.StatusUnauthorized, "FIN token not found in context", "PUT /fin/jobs/:job_id", "") + return + } + + route := "PUT /fin/jobs/" + g.Param("job_id") + jobID, err := uuid.Parse(g.Param("job_id")) + if err != nil { + apiError.SendErrorResponse(g, http.StatusBadRequest, "Failed to parse job ID", route, "") + return + } + + var request fin.ResultRequest + if err := g.ShouldBindJSON(&request); err != nil { + log.Error(err) + apiError.SendErrorResponse(g, http.StatusBadRequest, "Failed to parse job result", route, err.Error()) + return + } + + if request.State != fin.JobStateSuccess && request.State != fin.JobStateFailure { + apiError.SendErrorResponse(g, http.StatusBadRequest, "state must be \"success\" or \"failure\"", route, "") + return + } + + if err := h.workService.SubmitJobResult(g.Request.Context(), finToken, jobID, request.JobResult); err != nil { + h.sendJobError(g, route, err) + return + } + + g.Status(http.StatusNoContent) +} + +func (h *FinHandler) StatusPing(g *gin.Context) { + finToken, ok := h.getFinToken(g) + if !ok { + apiError.SendErrorResponse(g, http.StatusUnauthorized, "FIN token not found in context", "PATCH /fin/jobs/:job_id/status", "") + return + } + + route := "PATCH /fin/jobs/" + g.Param("job_id") + "/status" + jobID, err := uuid.Parse(g.Param("job_id")) + if err != nil { + apiError.SendErrorResponse(g, http.StatusBadRequest, "Failed to parse job ID", route, "") + return + } + + if g.Request.ContentLength > 0 { + var request fin.StatusPingRequest + if err := g.ShouldBindJSON(&request); err != nil { + log.Error(err) + apiError.SendErrorResponse(g, http.StatusBadRequest, "Failed to parse status ping", route, err.Error()) + return + } + } + + if err := h.workService.HeartbeatJob(g.Request.Context(), finToken, jobID); err != nil { + h.sendJobError(g, route, err) + return + } + + g.JSON(http.StatusOK, fin.StatusPingResponse{}) +} + +func (h *FinHandler) Unregister(g *gin.Context) { + finToken, ok := h.getFinToken(g) + if !ok { + apiError.SendErrorResponse(g, http.StatusUnauthorized, "FIN token not found in context", "DELETE /fin/", "") + return + } + + route := "DELETE /fin/" + if err := h.registry.UnregisterFin(g.Request.Context(), finToken); err != nil { + log.Error(err) + apiError.SendErrorResponse(g, http.StatusNotFound, "Fin not found", route, "") + return + } + + g.Status(http.StatusNoContent) +} + +func (h *FinHandler) List(g *gin.Context) { + records, err := h.registry.ListFins(g.Request.Context()) + if err != nil { + log.Error(err) + apiError.SendErrorResponse(g, http.StatusInternalServerError, "Failed to list fins", "GET /fin/", "") + return + } + g.JSON(http.StatusOK, fin.ListResponse{Fins: records}) +} + +func (h *FinHandler) Get(g *gin.Context) { + finID := g.Param("fin_id") + record, err := h.registry.GetFin(g.Request.Context(), finID) + if err != nil { + apiError.SendErrorResponse(g, http.StatusNotFound, "Fin not found", "GET /fin/"+finID, "") + return + } + g.JSON(http.StatusOK, record) +} + +func (h *FinHandler) Delete(g *gin.Context) { + finID := g.Param("fin_id") + route := "DELETE /fin/" + finID + if err := h.registry.DeleteFin(g.Request.Context(), finID); err != nil { + log.Error(err) + apiError.SendErrorResponse(g, http.StatusNotFound, "Fin not found", route, "") + return + } + g.Status(http.StatusNoContent) +} + +// ============================================================================ +// Helper methods +// ============================================================================ + +func (h *FinHandler) getFinToken(g *gin.Context) (string, bool) { + value, ok := g.Get(finContextKey) + if !ok { + return "", false + } + token, ok := value.(string) + return token, ok +} + +func (h *FinHandler) sendRegistrationError(g *gin.Context, route string, err error) { + log.Warning(err) + + var errRegistrationDisabled fin.ErrRegistrationDisabled + var errTokenInvalid fin.ErrRegistrationTokenInvalid + var errNoCapabilities fin.ErrNoCapabilities + var errCapabilityTypeEmpty fin.ErrCapabilityTypeEmpty + + switch { + case errors.As(err, &errRegistrationDisabled): + apiError.SendErrorResponse(g, http.StatusServiceUnavailable, "Fin registration is not configured", route, "") + case errors.As(err, &errTokenInvalid): + apiError.SendErrorResponse(g, http.StatusForbidden, err.Error(), route, "") + case errors.As(err, &errNoCapabilities): + apiError.SendErrorResponse(g, http.StatusBadRequest, "At least one capability is required", route, "") + case errors.As(err, &errCapabilityTypeEmpty): + apiError.SendErrorResponse(g, http.StatusBadRequest, "Every capability requires a non-empty type", route, "") + default: + apiError.SendErrorResponse(g, http.StatusInternalServerError, "Failed to register fin", route, "") + } +} + +func (h *FinHandler) sendJobError(g *gin.Context, route string, err error) { + log.Error(err) + + var notFound fin.ErrJobNotFound + var notLeased fin.ErrJobNotLeasedToFin + + switch { + case errors.As(err, ¬Found): + apiError.SendErrorResponse(g, http.StatusNotFound, "Job not found", route, "") + case errors.As(err, ¬Leased): + apiError.SendErrorResponse(g, http.StatusForbidden, "Job is not leased to this fin", route, "") + default: + apiError.SendErrorResponse(g, http.StatusInternalServerError, "Failed to process job request", route, "") + } +} + +func bearerToken(g *gin.Context) (string, bool) { + header := g.GetHeader("Authorization") + const prefix = "Bearer " + if !strings.HasPrefix(header, prefix) { + return "", false + } + value := strings.TrimPrefix(header, prefix) + if value == "" { + return "", false + } + return value, true +} diff --git a/internal/transport/http/handlers/fin/fin_api_test.go b/internal/transport/http/handlers/fin/fin_api_test.go new file mode 100644 index 00000000..f408fa0d --- /dev/null +++ b/internal/transport/http/handlers/fin/fin_api_test.go @@ -0,0 +1,444 @@ +package fin + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + finservice "soarca/internal/fins" + "soarca/internal/store" + "soarca/internal/store/storagetest" + "soarca/internal/workflow/capability/fin/queue" + finmodels "soarca/pkg/fins/protocol" + "soarca/test/unittest/mocks/mock_guid" + + "github.com/gin-gonic/gin" + "github.com/go-playground/assert/v2" + "github.com/google/uuid" +) + +const registrationToken = "test-registration-token" + +func newTestHandler(t *testing.T) (*FinHandler, storage.FinStore, *queue.Queue, *mock_guid.Mock_Guid) { + t.Helper() + repo := storagetest.New(t).Fins() + jobQueue := queue.New() + t.Cleanup(jobQueue.Close) + + fixedId := uuid.MustParse("11111111-1111-1111-1111-111111111111") + guidMock := new(mock_guid.Mock_Guid) + guidMock.On("New").Return(fixedId) + + registry := finservice.NewRegistry(repo, finservice.RegistryConfig{ + RegistrationToken: registrationToken, + StaleAfter: 2 * time.Minute, + }, guidMock) + + workService := finservice.NewWorkService(repo, jobQueue, finservice.WorkServiceConfig{ + LongPollTimeoutSeconds: 1, + JobLeaseSeconds: 60, + }) + + handler := NewFinHandler(registry, workService, Config{ + RegistrationToken: registrationToken, + PollIntervalSeconds: 5, + LongPollTimeoutSeconds: 1, + JobLeaseSeconds: 60, + StaleAfter: 2 * time.Minute, + }) + return handler, repo, jobQueue, guidMock +} + +func newTestRouter(handler *FinHandler) *gin.Engine { + gin.SetMode(gin.TestMode) + router := gin.New() + // Mirrors pkg/api.FinPublicRoutes + pkg/api.FinAdminRoutes: in + // production these are two separate route-registration functions + // (register/poll/jobs/status/unregister vs list/get), registered at two + // different points relative to the global admin auth middleware, so + // that Fin-token-authenticated calls are never also gated behind a + // soarca_admin JWT. That split doesn't affect this test harness (which + // has no admin auth middleware at all), so it's flattened here for + // convenience. + finRoutes := router.Group("/fin") + { + finRoutes.POST("/register", handler.Register) + finRoutes.GET("/", handler.List) + finRoutes.GET(":fin_id", handler.Get) + finRoutes.DELETE(":fin_id", handler.Delete) + + authenticated := finRoutes.Group("") + authenticated.Use(handler.RequireFinToken) + { + authenticated.POST("/poll", handler.Poll) + authenticated.PUT("jobs/:job_id", handler.SubmitResult) + authenticated.PATCH("jobs/:job_id/status", handler.StatusPing) + authenticated.DELETE("/", handler.Unregister) + } + } + return router +} + +func doRequest(router *gin.Engine, method string, path string, body any, bearer string) *httptest.ResponseRecorder { + var reader *bytes.Reader + if body != nil { + bodyBytes, _ := json.Marshal(body) + reader = bytes.NewReader(bodyBytes) + } else { + reader = bytes.NewReader(nil) + } + request := httptest.NewRequest(method, path, reader) + request.Header.Set("Content-Type", "application/json") + if bearer != "" { + request.Header.Set("Authorization", "Bearer "+bearer) + } + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, request) + return recorder +} + +func registerTestFin(t *testing.T, router *gin.Engine, capabilityType string) finmodels.RegisterResponse { + t.Helper() + recorder := doRequest(router, http.MethodPost, "/fin/register", finmodels.RegisterRequest{ + RegistrationToken: registrationToken, + DisplayName: "Test Fin", + Capabilities: []finmodels.Capability{{Type: capabilityType}}, + }, "") + if recorder.Code != http.StatusCreated { + t.Fatalf("expected 201, got %d: %s", recorder.Code, recorder.Body.String()) + } + var response finmodels.RegisterResponse + if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + return response +} + +func TestRegisterSucceeds(t *testing.T) { + handler, _, _, _ := newTestHandler(t) + router := newTestRouter(handler) + + response := registerTestFin(t, router, "pong") + assert.NotEqual(t, response.FinId, "") + assert.NotEqual(t, response.FinToken, "") + assert.Equal(t, response.PollIntervalSeconds, 5) +} + +func TestRegisterFailsWithWrongToken(t *testing.T) { + handler, _, _, _ := newTestHandler(t) + router := newTestRouter(handler) + + recorder := doRequest(router, http.MethodPost, "/fin/register", finmodels.RegisterRequest{ + RegistrationToken: "wrong-token", + Capabilities: []finmodels.Capability{{Type: "pong"}}, + }, "") + assert.Equal(t, recorder.Code, http.StatusForbidden) +} + +func TestRegisterFailsWithoutCapabilities(t *testing.T) { + handler, _, _, _ := newTestHandler(t) + router := newTestRouter(handler) + + recorder := doRequest(router, http.MethodPost, "/fin/register", finmodels.RegisterRequest{ + RegistrationToken: registrationToken, + Capabilities: []finmodels.Capability{}, + }, "") + assert.Equal(t, recorder.Code, http.StatusBadRequest) +} + +func TestRegisterFailsWhenNotConfigured(t *testing.T) { + repo := storagetest.New(t).Fins() + jobQueue := queue.New() + defer jobQueue.Close() + guidMock := new(mock_guid.Mock_Guid) + registry := finservice.NewRegistry(repo, finservice.RegistryConfig{ + RegistrationToken: "", + StaleAfter: 2 * time.Minute, + }, guidMock) + workService := finservice.NewWorkService(repo, jobQueue, finservice.WorkServiceConfig{ + LongPollTimeoutSeconds: 1, + JobLeaseSeconds: 60, + }) + handler := NewFinHandler(registry, workService, Config{}) + router := newTestRouter(handler) + + recorder := doRequest(router, http.MethodPost, "/fin/register", finmodels.RegisterRequest{ + RegistrationToken: "", + Capabilities: []finmodels.Capability{{Type: "pong"}}, + }, "") + assert.Equal(t, recorder.Code, http.StatusServiceUnavailable) +} + +func TestPollRequiresFinToken(t *testing.T) { + handler, _, _, _ := newTestHandler(t) + router := newTestRouter(handler) + + recorder := doRequest(router, http.MethodPost, "/fin/poll", nil, "") + assert.Equal(t, recorder.Code, http.StatusUnauthorized) + + recorder = doRequest(router, http.MethodPost, "/fin/poll", nil, "not-a-real-token") + assert.Equal(t, recorder.Code, http.StatusUnauthorized) +} + +func TestPollReturnsNoContentWhenNoJobIsAvailable(t *testing.T) { + handler, _, _, _ := newTestHandler(t) + router := newTestRouter(handler) + + registered := registerTestFin(t, router, "pong") + + recorder := doRequest(router, http.MethodPost, "/fin/poll", nil, registered.FinToken) + assert.Equal(t, recorder.Code, http.StatusNoContent) +} + +func TestPollReturnsEnqueuedJobAndUpdatesLastSeen(t *testing.T) { + handler, repo, jobQueue, _ := newTestHandler(t) + router := newTestRouter(handler) + + registered := registerTestFin(t, router, "pong") + before, err := repo.Get(context.Background(), registered.FinId) + if err != nil { + t.Fatal(err) + } + + job := finmodels.Job{ + JobId: uuid.New(), + CapabilityType: "pong", + LeaseExpiresInSeconds: 60, + } + go func() { + _, _ = jobQueue.Enqueue(context.Background(), job) + }() + time.Sleep(20 * time.Millisecond) + + recorder := doRequest(router, http.MethodPost, "/fin/poll", nil, registered.FinToken) + assert.Equal(t, recorder.Code, http.StatusOK) + + var response finmodels.PollResponse + if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + assert.Equal(t, response.Job.JobId, job.JobId) + + after, err := repo.Get(context.Background(), registered.FinId) + if err != nil { + t.Fatal(err) + } + if !after.LastSeen.After(before.LastSeen) { + t.Fatal("expected LastSeen to be updated by Poll") + } + + // Clean up: submit a result so the Enqueue goroutine doesn't leak past + // the test. + _ = jobQueue.Submit(job.JobId, registered.FinId, finmodels.JobResult{State: finmodels.JobStateSuccess}) +} + +func TestSubmitResultRoundTrip(t *testing.T) { + handler, repo, jobQueue, _ := newTestHandler(t) + router := newTestRouter(handler) + + registered := registerTestFin(t, router, "pong") + + job := finmodels.Job{ + JobId: uuid.New(), + CapabilityType: "pong", + LeaseExpiresInSeconds: 60, + } + resultCh := make(chan finmodels.JobResult, 1) + go func() { + result, _ := jobQueue.Enqueue(context.Background(), job) + resultCh <- result + }() + + recorder := doRequest(router, http.MethodPost, "/fin/poll", nil, registered.FinToken) + assert.Equal(t, recorder.Code, http.StatusOK) + + lastSeenAfterPoll, err := repo.Get(context.Background(), registered.FinId) + if err != nil { + t.Fatal(err) + } + time.Sleep(time.Millisecond) + + recorder = doRequest(router, http.MethodPut, "/fin/jobs/"+job.JobId.String(), + finmodels.ResultRequest{JobResult: finmodels.JobResult{State: finmodels.JobStateSuccess}}, registered.FinToken) + assert.Equal(t, recorder.Code, http.StatusNoContent) + + select { + case result := <-resultCh: + assert.Equal(t, result.State, finmodels.JobStateSuccess) + case <-time.After(time.Second): + t.Fatal("expected the enqueued job to receive its result") + } + + afterSubmit, err := repo.Get(context.Background(), registered.FinId) + if err != nil { + t.Fatal(err) + } + if !afterSubmit.LastSeen.After(lastSeenAfterPoll.LastSeen) { + t.Fatalf("expected submitting a job result to advance LastSeen: before=%v after=%v", + lastSeenAfterPoll.LastSeen, afterSubmit.LastSeen) + } +} + +func TestSubmitResultFailsForUnknownJob(t *testing.T) { + handler, _, _, _ := newTestHandler(t) + router := newTestRouter(handler) + + registered := registerTestFin(t, router, "pong") + + recorder := doRequest(router, http.MethodPut, "/fin/jobs/"+uuid.New().String(), + finmodels.ResultRequest{JobResult: finmodels.JobResult{State: finmodels.JobStateSuccess}}, registered.FinToken) + assert.Equal(t, recorder.Code, http.StatusNotFound) +} + +func TestSubmitResultFailsWhenLeasedToAnotherFin(t *testing.T) { + handler, _, jobQueue, guidMock := newTestHandler(t) + router := newTestRouter(handler) + + registeredA := registerTestFin(t, router, "pong") + + // A second fin registered under the same capability type, to claim the + // job first without being the one submitting the result. + guidMock.ExpectedCalls = nil + guidMock.On("New").Return(uuid.MustParse("22222222-2222-2222-2222-222222222222")) + registeredB := registerTestFin(t, router, "pong") + + job := finmodels.Job{JobId: uuid.New(), CapabilityType: "pong", LeaseExpiresInSeconds: 60} + go func() { _, _ = jobQueue.Enqueue(context.Background(), job) }() + + recorder := doRequest(router, http.MethodPost, "/fin/poll", nil, registeredB.FinToken) + assert.Equal(t, recorder.Code, http.StatusOK) + + recorder = doRequest(router, http.MethodPut, "/fin/jobs/"+job.JobId.String(), + finmodels.ResultRequest{JobResult: finmodels.JobResult{State: finmodels.JobStateSuccess}}, registeredA.FinToken) + assert.Equal(t, recorder.Code, http.StatusForbidden) +} + +func TestStatusPingExtendsLease(t *testing.T) { + handler, repo, jobQueue, _ := newTestHandler(t) + router := newTestRouter(handler) + + registered := registerTestFin(t, router, "pong") + + job := finmodels.Job{JobId: uuid.New(), CapabilityType: "pong", LeaseExpiresInSeconds: 60} + go func() { _, _ = jobQueue.Enqueue(context.Background(), job) }() + + recorder := doRequest(router, http.MethodPost, "/fin/poll", nil, registered.FinToken) + assert.Equal(t, recorder.Code, http.StatusOK) + + lastSeenAfterPoll, err := repo.Get(context.Background(), registered.FinId) + if err != nil { + t.Fatal(err) + } + time.Sleep(time.Millisecond) + + recorder = doRequest(router, http.MethodPatch, "/fin/jobs/"+job.JobId.String()+"/status", + finmodels.StatusPingRequest{Progress: "running"}, registered.FinToken) + assert.Equal(t, recorder.Code, http.StatusOK) + + var response finmodels.StatusPingResponse + if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + assert.Equal(t, response.Action, "") + + afterPing, err := repo.Get(context.Background(), registered.FinId) + if err != nil { + t.Fatal(err) + } + if !afterPing.LastSeen.After(lastSeenAfterPoll.LastSeen) { + t.Fatalf("expected a status ping to advance LastSeen: before=%v after=%v", + lastSeenAfterPoll.LastSeen, afterPing.LastSeen) + } + + _ = jobQueue.Submit(job.JobId, registered.FinId, finmodels.JobResult{State: finmodels.JobStateSuccess}) +} + +func TestUnregisterOwnRegistrationSucceeds(t *testing.T) { + handler, repo, _, _ := newTestHandler(t) + router := newTestRouter(handler) + + registered := registerTestFin(t, router, "pong") + + recorder := doRequest(router, http.MethodDelete, "/fin/", nil, registered.FinToken) + assert.Equal(t, recorder.Code, http.StatusNoContent) + + _, err := repo.Get(context.Background(), registered.FinId) + if err == nil { + t.Fatal("expected fin to be unregistered") + } +} + +func TestAdminDeleteRemovesAnyFinsRegistration(t *testing.T) { + handler, repo, _, _ := newTestHandler(t) + router := newTestRouter(handler) + + registered := registerTestFin(t, router, "pong") + + // Admin delete is not fin-token gated at all - no Authorization header. + recorder := doRequest(router, http.MethodDelete, "/fin/"+registered.FinId, nil, "") + assert.Equal(t, recorder.Code, http.StatusNoContent) + + _, err := repo.Get(context.Background(), registered.FinId) + if err == nil { + t.Fatal("expected fin to be unregistered") + } +} + +func TestListAndGet(t *testing.T) { + handler, _, _, _ := newTestHandler(t) + router := newTestRouter(handler) + + registered := registerTestFin(t, router, "pong") + + recorder := doRequest(router, http.MethodGet, "/fin/", nil, "") + assert.Equal(t, recorder.Code, http.StatusOK) + var listResponse finmodels.ListResponse + if err := json.Unmarshal(recorder.Body.Bytes(), &listResponse); err != nil { + t.Fatal(err) + } + assert.Equal(t, len(listResponse.Fins), 1) + assert.Equal(t, listResponse.Fins[0].Stale, false) + + recorder = doRequest(router, http.MethodGet, "/fin/"+registered.FinId, nil, "") + assert.Equal(t, recorder.Code, http.StatusOK) + var record finmodels.Record + if err := json.Unmarshal(recorder.Body.Bytes(), &record); err != nil { + t.Fatal(err) + } + assert.Equal(t, record.Stale, false) + + recorder = doRequest(router, http.MethodGet, "/fin/does-not-exist", nil, "") + assert.Equal(t, recorder.Code, http.StatusNotFound) +} + +func TestListAndGetMarkFinStaleAfterThreshold(t *testing.T) { + handler, repo, _, _ := newTestHandler(t) + handler.config.StaleAfter = time.Second + router := newTestRouter(handler) + + registered := registerTestFin(t, router, "pong") + if err := repo.Touch(context.Background(), registered.FinId, time.Now().Add(-time.Hour)); err != nil { + t.Fatal(err) + } + + recorder := doRequest(router, http.MethodGet, "/fin/"+registered.FinId, nil, "") + assert.Equal(t, recorder.Code, http.StatusOK) + var record finmodels.Record + if err := json.Unmarshal(recorder.Body.Bytes(), &record); err != nil { + t.Fatal(err) + } + assert.Equal(t, record.Stale, true) + + recorder = doRequest(router, http.MethodGet, "/fin/", nil, "") + assert.Equal(t, recorder.Code, http.StatusOK) + var listResponse finmodels.ListResponse + if err := json.Unmarshal(recorder.Body.Bytes(), &listResponse); err != nil { + t.Fatal(err) + } + assert.Equal(t, len(listResponse.Fins), 1) + assert.Equal(t, listResponse.Fins[0].Stale, true) +} diff --git a/internal/transport/http/handlers/fin/fin_e2e_test.go b/internal/transport/http/handlers/fin/fin_e2e_test.go new file mode 100644 index 00000000..c482e927 --- /dev/null +++ b/internal/transport/http/handlers/fin/fin_e2e_test.go @@ -0,0 +1,202 @@ +package fin_test + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + finservice "soarca/internal/fins" + "soarca/internal/store/storagetest" + "soarca/internal/transport/http/handlers/fin" + "soarca/internal/workflow/capability" + fincapability "soarca/internal/workflow/capability/fin" + "soarca/internal/workflow/capability/fin/queue" + "soarca/pkg/cacao" + finmodels "soarca/pkg/fins/protocol" + "soarca/internal/runs/model" + timeUtil "soarca/pkg/utils/time" + "soarca/test/unittest/mocks/mock_guid" + + "github.com/gin-gonic/gin" + "github.com/go-playground/assert/v2" + "github.com/google/uuid" +) + +// TestFullFinProtocolFlow is an end-to-end test of the whole Fin protocol +// stack wired together the same way internal/controller/controller.go +// wires it in production - a real fincapability.Capability enqueuing onto +// a real queue.Queue, and a real fin.FinHandler serving the actual +// register/poll/submit-result HTTP routes over that same queue - with no +// mocks standing in for either side. It exercises the full lifecycle this +// protocol exists for: register -> Execute() enqueues a job -> poll claims +// it -> submit result -> Execute() returns the result to the step machinery. +func TestFullFinProtocolFlow(t *testing.T) { + repo := storagetest.New(t).Fins() + jobQueue := queue.New() + t.Cleanup(jobQueue.Close) + + fixedFinId := uuid.MustParse("22222222-2222-2222-2222-222222222222") + finGuidMock := new(mock_guid.Mock_Guid) + finGuidMock.On("New").Return(fixedFinId) + + registry := finservice.NewRegistry(repo, finservice.RegistryConfig{ + RegistrationToken: "test-registration-token", + StaleAfter: 2 * time.Minute, + }, finGuidMock) + workService := finservice.NewWorkService(repo, jobQueue, finservice.WorkServiceConfig{ + LongPollTimeoutSeconds: 1, + JobLeaseSeconds: 60, + }) + handler := fin.NewFinHandler(registry, workService, fin.Config{ + RegistrationToken: "test-registration-token", + PollIntervalSeconds: 5, + LongPollTimeoutSeconds: 1, + JobLeaseSeconds: 60, + StaleAfter: 2 * time.Minute, + }) + + gin.SetMode(gin.TestMode) + router := gin.New() + finRoutes := router.Group("/fin") + { + finRoutes.POST("/register", handler.Register) + finRoutes.GET("/", handler.List) + finRoutes.GET(":fin_id", handler.Get) + finRoutes.DELETE(":fin_id", handler.Delete) + + authenticated := finRoutes.Group("") + authenticated.Use(handler.RequireFinToken) + { + authenticated.POST("/poll", handler.Poll) + authenticated.PUT("jobs/:job_id", handler.SubmitResult) + authenticated.PATCH("jobs/:job_id/status", handler.StatusPing) + authenticated.DELETE("/", handler.Unregister) + } + } + + // Register a Fin declaring the capability type the step below targets. + registerRecorder := doTestRequest(router, http.MethodPost, "/fin/register", finmodels.RegisterRequest{ + RegistrationToken: "test-registration-token", + DisplayName: "e2e-test-fin", + Capabilities: []finmodels.Capability{{Type: "custom-ssh-fin"}}, + }, "") + if registerRecorder.Code != http.StatusCreated { + t.Fatalf("expected 201 registering fin, got %d: %s", registerRecorder.Code, registerRecorder.Body.String()) + } + var registered finmodels.RegisterResponse + if err := json.Unmarshal(registerRecorder.Body.Bytes(), ®istered); err != nil { + t.Fatal(err) + } + + // Emulate the action executor's fallback capability, the same + // mechanism NewDecomposer() wires up in production (see + // action.Executor.SetFinFallback). + stepGuidMock := new(mock_guid.Mock_Guid) + jobId := uuid.MustParse("33333333-3333-3333-3333-333333333333") + stepGuidMock.On("New").Return(jobId) + finCap := fincapability.New(fincapability.Dependencies{ + Queue: jobQueue, + GUID: stepGuidMock, + Store: repo, + Time: &timeUtil.Time{}, + StaleAfter: 2 * time.Minute, + }) + + runId := uuid.MustParse("44444444-4444-4444-4444-444444444444") + metadata := run.Metadata{ + RunId: runId, + PlaybookId: "playbook--e2e", + StepId: "action--e2e", + StepRunId: uuid.MustParse("55555555-5555-5555-5555-555555555555"), + } + commandContext := capability.Context{ + Agent: cacao.AgentTarget{Type: "custom-ssh-fin"}, + Commands: []cacao.Command{{Type: "manual", Command: "sudo systemctl restart nginx"}}, + Step: cacao.Step{Timeout: 5000}, + } + + // Execute blocks (bounded by the step's timeout) until a Fin claims and + // resolves the job, so it must run concurrently with the poll/submit + // calls below - exactly like a real action executor waiting on a real + // external Fin process. + type executeOutcome struct { + variables cacao.Variables + err error + } + outcome := make(chan executeOutcome, 1) + go func() { + variables, err := finCap.Execute(metadata, commandContext) + outcome <- executeOutcome{variables, err} + }() + + // Poll claims the job Execute() just enqueued. The queue's Claim() + // blocks internally (see queue.Queue.Claim) until notified or the + // long-poll timeout elapses, so a single call here safely races with + // the Execute() goroutine above regardless of goroutine scheduling. + pollRecorder := doTestRequest(router, http.MethodPost, "/fin/poll", nil, registered.FinToken) + if pollRecorder.Code != http.StatusOK { + t.Fatalf("expected 200 polling for the job, got %d: %s", pollRecorder.Code, pollRecorder.Body.String()) + } + var pollResponse finmodels.PollResponse + if err := json.Unmarshal(pollRecorder.Body.Bytes(), &pollResponse); err != nil { + t.Fatal(err) + } + assert.Equal(t, pollResponse.Job.JobId, jobId) + assert.Equal(t, pollResponse.Job.RunId, runId) + assert.Equal(t, pollResponse.Job.CapabilityType, "custom-ssh-fin") + assert.Equal(t, len(pollResponse.Job.Commands), 1) + assert.Equal(t, pollResponse.Job.Commands[0].Command, "sudo systemctl restart nginx") + + // Submit the result, as the polling Fin would after actually running + // the command. + resultRecorder := doTestRequest(router, http.MethodPut, "/fin/jobs/"+jobId.String(), finmodels.ResultRequest{ + JobResult: finmodels.JobResult{ + State: finmodels.JobStateSuccess, + Variables: cacao.NewVariables(cacao.Variable{ + Type: "string", + Name: "__restarted__", + Value: "true", + }), + }, + }, registered.FinToken) + assert.Equal(t, resultRecorder.Code, http.StatusNoContent) + + select { + case result := <-outcome: + if result.err != nil { + t.Fatalf("expected Execute to succeed, got error: %v", result.err) + } + restarted, ok := result.variables["__restarted__"] + if !ok { + t.Fatal("expected __restarted__ variable to be returned from the fin result") + } + assert.Equal(t, restarted.Value, "true") + case <-time.After(2 * time.Second): + t.Fatal("Execute() did not return after the fin submitted its result") + } +} + +// doTestRequest is a standalone equivalent of fin_api_test.go's (internal, +// package fin) doRequest helper - this file lives in package fin_test (an +// external test package, needed to import fincapability without an import +// cycle), so it cannot reuse that unexported helper directly. +func doTestRequest(router *gin.Engine, method string, path string, body any, bearer string) *httptest.ResponseRecorder { + var reader *bytes.Reader + if body != nil { + bodyBytes, _ := json.Marshal(body) + reader = bytes.NewReader(bodyBytes) + } else { + reader = bytes.NewReader(nil) + } + request := httptest.NewRequest(method, path, reader) + request.Header.Set("Content-Type", "application/json") + if bearer != "" { + request.Header.Set("Authorization", "Bearer "+bearer) + } + recorder := httptest.NewRecorder() + router.ServeHTTP(recorder, request) + return recorder +} diff --git a/internal/transport/http/handlers/manual/manual_api.go b/internal/transport/http/handlers/manual/manual_api.go new file mode 100644 index 00000000..2fb74173 --- /dev/null +++ b/internal/transport/http/handlers/manual/manual_api.go @@ -0,0 +1,302 @@ +package manual + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "reflect" + "soarca/internal/logger" + "soarca/internal/orchestrator" + "soarca/internal/workflow/capability" + "soarca/internal/transport/http/schema" + "soarca/internal/manual/model" + "soarca/internal/runs/model" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + + apiError "soarca/internal/transport/http/handlers/error" +) + +var log *logger.Log + +type Empty struct{} + +func init() { + log = logger.Logger(reflect.TypeOf(Empty{}).PkgPath(), logger.Info, "", logger.Json) +} + +type ManualHandler struct { + inbox orchestrator.ManualInbox +} + +func NewManualHandler(inbox orchestrator.ManualInbox) *ManualHandler { + return &ManualHandler{inbox: inbox} +} + +// manual +// +// @Summary get all pending manual commands that still needs values to be returned +// @Schemes +// @Description get all pending manual commands that still needs values to be returned +// @Tags manual +// @Accept json +// @Produce json +// @Success 200 {object} []api.PendingCommandData +// @failure 400 {object} []api.PendingCommandData +// @Router /manual/ [GET] +func (manualHandler *ManualHandler) GetPendingCommands(g *gin.Context) { + commands, err := manualHandler.inbox.ListPendingCommands() + if err != nil { + log.Error(err) + apiError.SendErrorResponse(g, http.StatusInternalServerError, + "Failed get pending manual commands", + "GET /manual/", "") + return + } + + response := []api.PendingCommandData{} + for _, command := range commands { + response = append(response, manualHandler.parseCommandInfoToResponse(command)) + } + + g.JSON(http.StatusOK, + response) +} + +// manual +// +// @Summary get a specific manual command that still needs a value to be returned +// @Schemes +// @Description get a specific manual command that still needs a value to be returned +// @Tags manual +// @Accept json +// @Produce json +// @Param run_id path string true "run ID" +// @Param step_run_id path string true "step run ID (identifies a specific pending step invocation; see GET /manual/ to discover it, as multiple pending commands may share the same step ID)" +// @Success 200 {object} api.PendingCommandData +// @failure 400 {object} api.Error +// @Router /manual/{run_id}/{step_run_id} [GET] +func (manualHandler *ManualHandler) GetPendingCommand(g *gin.Context) { + runIdParam := g.Param("run_id") + stepRunIdParam := g.Param("step_run_id") + execId, err := uuid.Parse(runIdParam) + if err != nil { + log.Error(err) + apiError.SendErrorResponse(g, http.StatusBadRequest, + "Failed to parse run ID", + "GET /manual/"+runIdParam+"/"+stepRunIdParam, "") + return + } + stepRunId, err := uuid.Parse(stepRunIdParam) + if err != nil { + log.Error(err) + apiError.SendErrorResponse(g, http.StatusBadRequest, + "Failed to parse step run ID", + "GET /manual/"+runIdParam+"/"+stepRunIdParam, "") + return + } + + commandData, err := manualHandler.inbox.GetPendingCommand(run.Metadata{RunId: execId, StepRunId: stepRunId}) + if err != nil { + log.Error(err) + code := http.StatusBadRequest + if errors.Is(err, manual.ErrorPendingCommandNotFound{}) { + code = http.StatusNotFound + } + apiError.SendErrorResponse(g, code, + "Failed to provide pending manual command", + "GET /manual/"+runIdParam+"/"+stepRunIdParam, "") + return + } + + commandInfo := manualHandler.parseCommandInfoToResponse(commandData) + + g.JSON(http.StatusOK, commandInfo) +} + +// manual +// +// @Summary resolve a specific pending manual command by supplying its out args +// @Schemes +// @Description resolve a specific pending manual command by supplying its out args. This is a PUT +// @Description on the same resource GET /manual/{run_id}/{step_run_id} identifies, not a +// @Description generic RPC-style action, so the ids live in the path, not the body. +// @Tags manual +// @Accept json +// @Produce json +// @Param run_id path string true "run ID" +// @Param step_run_id path string true "step run ID (identifies a specific pending step invocation; see GET /manual/ to discover it, as multiple pending commands may share the same step ID)" +// @Param data body api.ManualOutArgsUpdatePayload true "resolution" +// @Success 200 {object} api.RunStarted +// @failure 400 {object} api.Error +// @Router /manual/{run_id}/{step_run_id} [PUT] +func (manualHandler *ManualHandler) PutContinue(g *gin.Context) { + runIdParam := g.Param("run_id") + stepRunIdParam := g.Param("step_run_id") + route := "PUT /manual/" + runIdParam + "/" + stepRunIdParam + + execId, err := uuid.Parse(runIdParam) + if err != nil { + log.Error(err) + apiError.SendErrorResponse(g, http.StatusBadRequest, + "Failed to parse run ID", + route, "") + return + } + stepRunId, err := uuid.Parse(stepRunIdParam) + if err != nil { + log.Error(err) + apiError.SendErrorResponse(g, http.StatusBadRequest, + "Failed to parse step run ID", + route, "") + return + } + byteData, err := io.ReadAll(g.Request.Body) + if err != nil { + log.Error("failed") + apiError.SendErrorResponse(g, http.StatusBadRequest, + "Failed to read json", + route, "") + return + } + + outArgsUpdate, err := manualHandler.parseManualOutArgsUpdate(byteData) + if err != nil { + apiError.SendErrorResponse(g, http.StatusBadRequest, + fmt.Sprint(fmt.Errorf("failed to parse manual out args payload: %w", err)), + route, err.Error()) + return + } + + // Looked up here (rather than only implicitly inside PostContinue below) + // so the response can report the PlaybookId, and so an unknown resource + // is reported before any out-args validation runs against it. + pendingCommand, err := manualHandler.inbox.GetPendingCommand(run.Metadata{RunId: execId, StepRunId: stepRunId}) + if err != nil { + log.Error(err) + code := http.StatusBadRequest + if errors.Is(err, manual.ErrorPendingCommandNotFound{}) { + code = http.StatusNotFound + } + apiError.SendErrorResponse(g, code, + "Pending manual command not found", + route, "") + return + } + + response := manualHandler.parseManualOutArgsToResponse(pendingCommand.Metadata, outArgsUpdate) + + err = manualHandler.inbox.ContinuePendingCommand(response) + if err != nil { + log.Error(err) + code := http.StatusBadRequest + msg := "Failed to post the continue request" + if errors.Is(err, manual.ErrorPendingCommandNotFound{}) { + code = http.StatusNotFound + msg = "Pending command not found" + } else if errors.Is(err, manual.ErrorNonMatchingOutArgs{}) { + code = http.StatusBadRequest + msg = "Provided out args don't match with expected" + } + apiError.SendErrorResponse(g, code, + msg, + route, "") + return + } + + g.JSON( + http.StatusOK, + api.RunStarted{ + RunId: execId, + PlaybookId: pendingCommand.Metadata.PlaybookId, + }) +} + +// ############################################################################ +// Utility +// ############################################################################ + +func (manualHandler *ManualHandler) parseManualOutArgsUpdate(postData []byte) (api.ManualOutArgsUpdatePayload, error) { + decoder := json.NewDecoder(bytes.NewReader(postData)) + decoder.DisallowUnknownFields() + var outArgsUpdate api.ManualOutArgsUpdatePayload + err := decoder.Decode(&outArgsUpdate) + if err != nil { + errorString := fmt.Errorf("failed to unmarshal JSON: %w", err) + log.Error(errorString) + return api.ManualOutArgsUpdatePayload{}, errorString + } + + // Check if variable names match + for varName, variable := range outArgsUpdate.ResponseOutArgs { + if varName != variable.Name { + errorString := fmt.Errorf( + "variable name mismatch for variable %s: has different name property: %s", + varName, variable.Name) + log.Error(errorString) + return api.ManualOutArgsUpdatePayload{}, errorString + } + } + + return outArgsUpdate, nil +} + +func (manualHandler *ManualHandler) parseCommandInfoToResponse(commandInfo manual.CommandInfo) api.PendingCommandData { + // Manual is a human-resolved, single-outcome step (one response resolves + // the whole pending entry), but a step may list multiple commands and + // targets -- surface all of them rather than only the first. Multiple + // pending commands may share the same StepId (e.g. overlapping + // while-loop iterations); each is a distinct entry here, disambiguated + // by StepRunId. + commands := make([]api.ManualCommand, 0, len(commandInfo.Context.Commands)) + for _, command := range commandInfo.Context.Commands { + commandText := command.Command + isBase64 := false + if len(command.CommandB64) > 0 { + commandText = command.CommandB64 + isBase64 = true + } + commands = append(commands, api.ManualCommand{ + Description: command.Description, + Command: commandText, + CommandIsBase64: isBase64, + }) + } + + targets := make([]capability.ResolvedTarget, 0, len(commandInfo.Context.Targets)) + for _, resolvedTarget := range commandInfo.Context.Targets { + targets = append(targets, capability.ResolvedTarget{ + Target: resolvedTarget.Target, + Authentication: resolvedTarget.Authentication, + }) + } + + response := api.PendingCommandData{ + Type: "manual-command-info", + RunId: commandInfo.Metadata.RunId.String(), + PlaybookId: commandInfo.Metadata.PlaybookId, + StepId: commandInfo.Metadata.StepId, + StepRunId: commandInfo.Metadata.StepRunId.String(), + Commands: commands, + Targets: targets, + OutVariables: commandInfo.OutArgsVariables, + } + + return response +} + +func (manualHandler *ManualHandler) parseManualOutArgsToResponse( + metadata run.Metadata, + response api.ManualOutArgsUpdatePayload, +) manual.Response { + return manual.Response{ + Metadata: metadata, + ResponseStatus: response.ResponseStatus, + OutArgsVariables: response.ResponseOutArgs, + ResponseError: nil, + } +} diff --git a/internal/transport/http/handlers/manual/manual_api_utils_test.go b/internal/transport/http/handlers/manual/manual_api_utils_test.go new file mode 100644 index 00000000..6a3bcbd3 --- /dev/null +++ b/internal/transport/http/handlers/manual/manual_api_utils_test.go @@ -0,0 +1,168 @@ +package manual + +import ( + "errors" + "reflect" + "soarca/internal/workflow/capability" + "soarca/internal/transport/http/schema" + "soarca/pkg/cacao" + "soarca/internal/manual/model" + "soarca/internal/runs/model" + "soarca/test/unittest/mocks/mock_manual_inbox_storage" + "testing" + + "github.com/go-playground/assert/v2" + "github.com/google/uuid" +) + +func TestParseManualOutArgsUpdate(t *testing.T) { + manualHandler := NewManualHandler(&mock_manual_inbox_storage.MockInboxStorage{}) + + jsonPayload := `{"type":"out-args-update","response_status":"success","response_out_args":{"__test__":{"type":"string","name":"__test__","value":"updated!"}}}` + bytesPayload := []byte(jsonPayload) + + outVariable := cacao.Variable{Type: "string", Name: "__test__", Value: "updated!"} + outVariables := map[string]cacao.Variable{"__test__": outVariable} + + expectedPayload := api.ManualOutArgsUpdatePayload{ + Type: "out-args-update", + ResponseStatus: manual.ManualResponseSuccessStatus, + ResponseOutArgs: outVariables, + } + + receivedPayload, err := manualHandler.parseManualOutArgsUpdate(bytesPayload) + if err != nil { + t.Fatalf("failed to parse manual out args update: %v", err) + } + assert.Equal(t, receivedPayload, expectedPayload) +} + +func TestParseManualOutArgsUpdateFailOnVariablesNames(t *testing.T) { + manualHandler := NewManualHandler(&mock_manual_inbox_storage.MockInboxStorage{}) + + jsonPayload := `{"type":"out-args-update","response_status":"success","response_out_args":{"__test__":{"type":"string","name":"__wrong_name__","value":"updated!"}}}` + bytesPayload := []byte(jsonPayload) + + expecedErr := errors.New("variable name mismatch for variable __test__: has different name property: __wrong_name__") + _, err := manualHandler.parseManualOutArgsUpdate(bytesPayload) + if err == nil { + t.Log("an error for non-matching variables names should have been raised") + t.Fail() + } + + assert.Equal(t, err, expecedErr) +} + +func TestParseManualOutArgsUpdateFailOnInvalidModel(t *testing.T) { + manualHandler := NewManualHandler(&mock_manual_inbox_storage.MockInboxStorage{}) + + jsonPayload := `{"invalidProperty":"out-args-update","response_status":"success","response_out_args":{"__test__":{"type":"string","name":"__wrong_name__","value":"updated!"}}}` + bytesPayload := []byte(jsonPayload) + + expectedErr := "failed to unmarshal JSON: json: unknown field \"invalidProperty\"" + _, err := manualHandler.parseManualOutArgsUpdate(bytesPayload) + if err == nil { + t.Log("an error for non-matching variables names should have been raised") + t.Fail() + } + + assert.Equal(t, err.Error(), expectedErr) +} + +func TestParseCommandInfoToResponseIncludesAllCommandsAndTargets(t *testing.T) { + + manualHandler := NewManualHandler(&mock_manual_inbox_storage.MockInboxStorage{}) + + testExecId := "50b6d52c-6efc-4516-a242-dfbc5c89d421" + testStepId := "61a4d52c-6efc-4516-a242-dfbc5c89d312" + testPlaybookId := "21a4d52c-6efc-4516-a242-dfbc5c89d312" + testStepExecId := "71a4d52c-6efc-4516-a242-dfbc5c89d999" + + commandOne := cacao.Command{Type: "manual", Command: "please do a test thanks", Description: "testing!"} + commandTwo := cacao.Command{Type: "manual", CommandB64: "cGxlYXNlIGRvIGFub3RoZXIgdGVzdA==", Description: "testing again!"} + targetOne := cacao.AgentTarget{Type: "target", Name: "myself"} + targetTwo := cacao.AgentTarget{Type: "target", Name: "someoneelse"} + authOne := cacao.AuthenticationInformation{Type: "user-auth", Username: "operator", Password: "hunter2"} + variable2 := cacao.Variable{Type: "string", Name: "__test__", Value: "some value"} + inputVariable := map[string]cacao.Variable{"__test__": variable2} + + context := capability.Context{ + Commands: []cacao.Command{commandOne, commandTwo}, + Targets: []capability.ResolvedTarget{ + {Target: targetOne, Authentication: authOne}, + {Target: targetTwo}, + }, + Variables: inputVariable, + } + + testVariables := cacao.NewVariables(cacao.Variable{Type: "string", Name: "__test__", Value: "test!"}) + + commandInfo := manual.CommandInfo{ + Metadata: run.Metadata{ + PlaybookId: testPlaybookId, + RunId: uuid.MustParse(testExecId), + StepId: testStepId, + StepRunId: uuid.MustParse(testStepExecId)}, + Context: context, + OutArgsVariables: testVariables, + } + + expectedInteractionCommand := api.PendingCommandData{ + Type: "manual-command-info", + RunId: testExecId, + PlaybookId: testPlaybookId, + StepId: testStepId, + StepRunId: testStepExecId, + Commands: []api.ManualCommand{ + {Description: "testing!", Command: "please do a test thanks", CommandIsBase64: false}, + {Description: "testing again!", Command: "cGxlYXNlIGRvIGFub3RoZXIgdGVzdA==", CommandIsBase64: true}, + }, + Targets: []capability.ResolvedTarget{ + {Target: targetOne, Authentication: authOne}, + {Target: targetTwo}, + }, + OutVariables: testVariables, + } + + returnPendingCommandData := manualHandler.parseCommandInfoToResponse(commandInfo) + t.Log(returnPendingCommandData) + t.Log(expectedInteractionCommand) + + assert.Equal(t, reflect.DeepEqual(returnPendingCommandData, expectedInteractionCommand), true) +} + +func TestParseManualOutArgsToResponse(t *testing.T) { + manualHandler := NewManualHandler(&mock_manual_inbox_storage.MockInboxStorage{}) + + testExecId := "50b6d52c-6efc-4516-a242-dfbc5c89d421" + testStepId := "61a4d52c-6efc-4516-a242-dfbc5c89d312" + testPlaybookId := "21a4d52c-6efc-4516-a242-dfbc5c89d312" + testStepExecId := "71a4d52c-6efc-4516-a242-dfbc5c89d999" + + metadata := run.Metadata{ + PlaybookId: testPlaybookId, + RunId: uuid.MustParse(testExecId), + StepId: testStepId, + StepRunId: uuid.MustParse(testStepExecId), + } + + outVariable := cacao.Variable{Type: "string", Name: "__test__", Value: "updated!"} + outVariables := map[string]cacao.Variable{"__test__": outVariable} + + payload := api.ManualOutArgsUpdatePayload{ + Type: "out-args-update", + ResponseStatus: manual.ManualResponseFailureStatus, + ResponseOutArgs: outVariables, + } + + expetedResponse := manual.Response{ + Metadata: metadata, + ResponseStatus: manual.ManualResponseFailureStatus, + OutArgsVariables: outVariables, + ResponseError: nil, + } + + response := manualHandler.parseManualOutArgsToResponse(metadata, payload) + + assert.Equal(t, expetedResponse, response) +} diff --git a/internal/transport/http/handlers/middleware/gin_log_middleware.go b/internal/transport/http/handlers/middleware/gin_log_middleware.go new file mode 100644 index 00000000..41ff3b41 --- /dev/null +++ b/internal/transport/http/handlers/middleware/gin_log_middleware.go @@ -0,0 +1,32 @@ +package loggerfactory + +import ( + "time" + + gin "github.com/gin-gonic/gin" + logrus "github.com/sirupsen/logrus" +) + +func LoggingMiddleware(fl *logrus.Logger) gin.HandlerFunc { + return func(ctx *gin.Context) { + + startTime := time.Now() + ctx.Next() + endTime := time.Now() + latencyTime := endTime.Sub(startTime) + reqMethod := ctx.Request.Method + reqUri := ctx.Request.RequestURI + statusCode := ctx.Writer.Status() + clientIP := ctx.ClientIP() + + fl.WithFields(logrus.Fields{ + "METHOD": reqMethod, + "URI": reqUri, + "STATUS": statusCode, + "LATENCY": latencyTime, + "CLIENT_IP": clientIP, + }).Info("HTTP REQUEST") + + ctx.Next() + } +} diff --git a/internal/transport/http/handlers/playbook/playbook_api.go b/internal/transport/http/handlers/playbook/playbook_api.go new file mode 100644 index 00000000..cdc06871 --- /dev/null +++ b/internal/transport/http/handlers/playbook/playbook_api.go @@ -0,0 +1,139 @@ +package playbook + +import ( + "encoding/json" + "io" + "net/http" + "reflect" + "soarca/internal/logger" + "soarca/internal/orchestrator" + "soarca/internal/store" + "soarca/pkg/cacao" + "strconv" + + "github.com/gin-gonic/gin" +) + +var log *logger.Log + +type Empty struct{} + +func init() { + log = logger.Logger(reflect.TypeOf(Empty{}).PkgPath(), logger.Info, "", logger.Json) +} + +type playbookHandler struct { + playbooks orchestrator.PlaybookService +} + +func NewPlaybookHandler(playbooks orchestrator.PlaybookService) *playbookHandler { + return &playbookHandler{playbooks: playbooks} +} + +func (handler *playbookHandler) GetAllPlaybooks(g *gin.Context) { + log.Trace("Trying to obtain all playbook IDs") + + returnListIDs, err := handler.playbooks.ListPlaybooks(g.Request.Context()) + if err != nil { + log.Debug("Could not obtain any Playbooks", err) + SendErrorResponse(g, http.StatusBadRequest, "Could not obtain any IDs", "GET /playbook") + return + } + + g.JSON(http.StatusOK, returnListIDs) +} + +func (handler *playbookHandler) GetAllPlaybookMetas(g *gin.Context) { + log.Trace("Trying to obtain all playbook IDs") + + returnListIDs, err := handler.playbooks.ListPlaybookMetas(g.Request.Context()) + if err != nil { + log.Debug("Could not obtain any PlaybookMetas", err) + SendErrorResponse(g, http.StatusBadRequest, "Could not obtain any IDs", "GET /playbook/meta") + return + } + + g.JSON(http.StatusOK, returnListIDs) +} + +func (handler *playbookHandler) SubmitPlaybook(g *gin.Context) { + jsonData, err := io.ReadAll(g.Request.Body) + if err != nil { + log.Trace("Submit playbook Endpoint has failed: ", err.Error()) + SendErrorResponse(g, http.StatusBadRequest, "Failed to marshall json on server side", "POST /playbook") + return + } + var playbook cacao.Playbook + if err := json.Unmarshal(jsonData, &playbook); err != nil { + SendErrorResponse(g, http.StatusBadRequest, "Could not create playbook. Is the playbook correct?", "POST /playbook") + return + } + if err := handler.playbooks.CreatePlaybook(g.Request.Context(), &playbook); err != nil { + if err == storage.ErrConflict { + SendErrorResponse(g, http.StatusConflict, "Provided duplicate playbook, already in database", "POST /playbook") + return + } + SendErrorResponse(g, http.StatusBadRequest, "Could not create playbook. Is the playbook correct?", "POST /playbook") + return + } + g.JSON(http.StatusCreated, playbook) +} + +func (handler *playbookHandler) GetPlaybookByID(g *gin.Context) { + id := g.Param("id") + log.Trace("Trying to obtain playbook for id: ", id) + + playbook, err := handler.playbooks.GetPlaybook(g.Request.Context(), id) + if err != nil { + log.Debug("Could not find document for given id") + SendErrorResponse(g, http.StatusNotFound, "Could not find playbook for given ID", "GET /playbook/{id}") + return + } + g.JSON(http.StatusOK, playbook) +} + +func (handler *playbookHandler) UpdatePlaybookByID(g *gin.Context) { + id := g.Param("id") + log.Trace("Trying to update playbook for id: ", id) + + jsonData, err := io.ReadAll(g.Request.Body) + if err != nil { + log.Debug("Update playbook Endpoint has failed: ", err.Error()) + SendErrorResponse(g, http.StatusBadRequest, "Failed to marshall json on server sider", "PUT /playbook/{id}") + return + } + var updatedPlaybook cacao.Playbook + if err := json.Unmarshal(jsonData, &updatedPlaybook); err != nil { + SendErrorResponse(g, http.StatusBadRequest, "Could not find playbook for given ID", "PUT /playbook/{id}") + return + } + if err := handler.playbooks.UpdatePlaybook(g.Request.Context(), id, &updatedPlaybook); err != nil { + if err == storage.ErrNotFound { + SendErrorResponse(g, http.StatusNotFound, "Could not find playbook for given ID", "PUT /playbook/{id}") + return + } + SendErrorResponse(g, http.StatusBadRequest, "Could not find playbook for given ID", "PUT /playbook/{id}") + return + } + g.JSON(http.StatusOK, updatedPlaybook) +} + +func (handler *playbookHandler) DeleteByPlaybookID(g *gin.Context) { + id := g.Param("id") + err := handler.playbooks.DeletePlaybook(g.Request.Context(), id) + if err != nil { + log.Debug("Something when wrong tying to delete the playbook object. Does the object exists?") + SendErrorResponse(g, http.StatusBadRequest, "Could not delete object", "DELETE /playbook/{id}") + return + } + g.Status(http.StatusOK) +} + +func SendErrorResponse(g *gin.Context, status int, message string, orginal_call string) { + msg := gin.H{ + "status": strconv.Itoa(status), + "message": message, + "original-call": orginal_call, + } + g.JSON(status, msg) +} diff --git a/internal/transport/http/handlers/reporter/reporter_api.go b/internal/transport/http/handlers/reporter/reporter_api.go new file mode 100644 index 00000000..e88e1ecd --- /dev/null +++ b/internal/transport/http/handlers/reporter/reporter_api.go @@ -0,0 +1,105 @@ +package reporter + +import ( + "net/http" + "reflect" + "soarca/internal/logger" + execsvc "soarca/internal/runs" + "soarca/internal/transport/http/handlers/error" + api "soarca/internal/transport/http/schema" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" +) + +var log *logger.Log + +type Empty struct{} + +func init() { + log = logger.Logger(reflect.TypeOf(Empty{}).PkgPath(), logger.Info, "", logger.Json) +} + +// reportHandler implements the handler functions that can be called by the gin api is dependent on a database. +type reportHandler struct { + runs execsvc.Runner +} + +// NewReportHandler makes a new instance of playbookControler +func NewReportHandler(runner execsvc.Runner) *reportHandler { + return &reportHandler{runs: runner} +} + +// GetRuns GET handler for obtaining all the runs that can be retrieved. +// Returns this to the gin context as a list if run IDs in json format +// +// @Summary gets all the UUIDs for the runs that can be retireved +// @Schemes +// @Description return all stored runs +// @Tags reporter +// @Produce json +// @success 200 {array} api.PlaybookRunReport +// @failure 400 {object} api.Error +// @Router /reporter [GET] +func (reportHandler *reportHandler) GetRuns(g *gin.Context) { + runs, err := reportHandler.runs.List(g.Request.Context()) + if err != nil { + log.Debug("Could not get runs from run state") + error.SendErrorResponse(g, http.StatusInternalServerError, "Could not get runs", "GET /reporter/", "") + return + } + + runsParsed := []api.PlaybookRunReport{} + for _, runEntry := range runs { + runEntryParsed, err := parseRunStateEntry(runEntry) + if err != nil { + log.Debug("Could not parse entry to reporter result model") + log.Error(err) + error.SendErrorResponse(g, http.StatusInternalServerError, "Could not parse run report", "GET /reporter/", "") + return + } + runsParsed = append(runsParsed, runEntryParsed) + } + + g.JSON(http.StatusOK, runsParsed) +} + +// GetRunReport GET handler for obtaining the information about an run. +// Returns this to the gin context as a PlaybookRunReport object at soarca/model/api/reporter +// +// @Summary gets information about an ongoing playbook run +// @Schemes +// @Description return run information +// @Tags reporter +// @Produce json +// @Param id path string true "run identifier" +// @success 200 {object} api.PlaybookRunReport +// @failure 400 {object} api.Error +// @Router /reporter/{id} [GET] +func (handler *reportHandler) GetRunReport(g *gin.Context) { + id := g.Param("id") + log.Trace("Trying to obtain run for id: ", id) + uuid, err := uuid.Parse(id) + if err != nil { + log.Debug("Could not parse id parameter for request") + error.SendErrorResponse(g, http.StatusBadRequest, "Could not parse id parameter for request", "GET /reporter/"+id, err.Error()) + return + } + + runEntry, err := handler.runs.Report(g.Request.Context(), uuid) + if err != nil { + log.Debug("Could not find run for given id") + log.Error(err) + error.SendErrorResponse(g, http.StatusBadRequest, "Could not find run for given ID", "GET /reporter/"+id, "") + return + } + + runEntryParsed, err := parseRunStateEntry(runEntry) + if err != nil { + log.Debug("Could not parse entry to reporter result model") + log.Error(err) + error.SendErrorResponse(g, http.StatusInternalServerError, "Could not parse run report", "GET /reporter/"+id, "") + return + } + g.JSON(http.StatusOK, runEntryParsed) +} diff --git a/internal/transport/http/handlers/reporter/reporter_parser.go b/internal/transport/http/handlers/reporter/reporter_parser.go new file mode 100644 index 00000000..78a89b15 --- /dev/null +++ b/internal/transport/http/handlers/reporter/reporter_parser.go @@ -0,0 +1,74 @@ +package reporter + +import ( + api_model "soarca/internal/transport/http/schema" + runstate_model "soarca/internal/runs/state" +) + +const defaultRequestInterval int = 5 + +func parseRunStateEntry(entry runstate_model.RunEntry) (api_model.PlaybookRunReport, error) { + playbookStatus := api_model.RunStatusEnum2String(entry.Status) + + playbookStatusText, err := api_model.GetRunStatusText(playbookStatus, api_model.ReportLevelPlaybook) + if err != nil { + return api_model.PlaybookRunReport{}, err + } + if entry.Error != nil { + playbookStatusText = playbookStatusText + " - error: " + entry.Error.Error() + } + + stepResults, err := parseRunStateSteps(entry.StepResults) + if err != nil { + return api_model.PlaybookRunReport{}, err + } + + runReport := api_model.PlaybookRunReport{ + Type: "run_status", + Name: entry.Name, + Description: entry.Description, + RunId: entry.RunId.String(), + PlaybookId: entry.PlaybookId, + Started: entry.Started, + Ended: entry.Ended, + Status: playbookStatus, + StatusText: playbookStatusText, + StepResults: stepResults, + RequestInterval: defaultRequestInterval, + } + return runReport, nil +} + +func parseRunStateSteps(stepEntries map[string]runstate_model.StepResult) (map[string]api_model.StepRunReport, error) { + parsedEntries := map[string]api_model.StepRunReport{} + for stepRunKey, stepEntry := range stepEntries { + + stepStatus := api_model.RunStatusEnum2String(stepEntry.Status) + + stepStatusText, err := api_model.GetRunStatusText(stepStatus, api_model.ReportLevelStep) + if err != nil { + return map[string]api_model.StepRunReport{}, err + } + + if stepEntry.Error != nil { + stepStatusText = stepStatusText + " - error: " + stepEntry.Error.Error() + } + + parsedEntries[stepRunKey] = api_model.StepRunReport{ + RunId: stepEntry.RunId.String(), + StepId: stepEntry.StepId, + StepRunId: stepEntry.StepRunId.String(), + Name: stepEntry.Name, + Description: stepEntry.Description, + Started: stepEntry.Started, + Ended: stepEntry.Ended, + Status: stepStatus, + StatusText: stepStatusText, + ExecutedBy: "soarca", + CommandsB64: stepEntry.CommandsB64, + Variables: stepEntry.Variables, + AutomatedRun: stepEntry.IsAutomated, + } + } + return parsedEntries, nil +} diff --git a/internal/transport/http/handlers/routes.go b/internal/transport/http/handlers/routes.go new file mode 100644 index 00000000..40ba7afc --- /dev/null +++ b/internal/transport/http/handlers/routes.go @@ -0,0 +1,147 @@ +package api + +import ( + "reflect" + open_api "soarca/api" + "soarca/internal/logger" + "soarca/internal/runs" + "soarca/internal/orchestrator" + fin_handler "soarca/internal/transport/http/handlers/fin" + manual_handler "soarca/internal/transport/http/handlers/manual" + playbook_handler "soarca/internal/transport/http/handlers/playbook" + reporter_handler "soarca/internal/transport/http/handlers/reporter" + status_handler "soarca/internal/transport/http/handlers/status" + trigger_handler "soarca/internal/transport/http/handlers/trigger" + + "github.com/gin-contrib/cors" + gin "github.com/gin-gonic/gin" + swaggerfiles "github.com/swaggo/files" + ginSwagger "github.com/swaggo/gin-swagger" +) + +var log *logger.Log + +type Empty struct{} + +func init() { + log = logger.Logger(reflect.TypeOf(Empty{}).PkgPath(), logger.Info, "", logger.Json) +} + +// ============================================================================ +// Handler constructors (for callers that inject services directly) +// ============================================================================ + +func NewTriggerHandler(runner runs.Runner) *trigger_handler.TriggerHandler { + return trigger_handler.NewTriggerHandler(runner) +} + +func NewManualHandler(inbox orchestrator.ManualInbox) *manual_handler.ManualHandler { + return manual_handler.NewManualHandler(inbox) +} + +// ============================================================================ +// Route registration functions +// ============================================================================ + +func Cors(app *gin.Engine, origins []string) { + config := cors.DefaultConfig() + config.AllowOrigins = origins + app.Use(cors.New(config)) +} + +func Logging(app *gin.Engine) {} + +func Swagger(app *gin.Engine) { swaggerRoutes(app) } + +func swaggerRoutes(route *gin.Engine) { + open_api.SwaggerInfo.BasePath = "/" + swaggerRoutes := route.Group("/swagger") + { + swaggerRoutes.GET("/*any", ginSwagger.WrapHandler(swaggerfiles.Handler)) + } +} + +func StatusRoutes(route *gin.Engine) { + router := route.Group("/status") + { + router.GET("/", status_handler.GetApi) + router.GET("/ping", status_handler.GetPong) + } +} + +func TriggerRoutes(route *gin.Engine, triggerHandler *trigger_handler.TriggerHandler) { + triggerRoutes := route.Group("/trigger") + { + triggerRoutes.POST("/playbook", triggerHandler.Execute) + triggerRoutes.POST("/playbook/:id", triggerHandler.ExecuteById) + } +} + +func ManualRoutes(route *gin.Engine, manualHandler *manual_handler.ManualHandler) { + manualRoutes := route.Group("/manual") + { + manualRoutes.GET("/", manualHandler.GetPendingCommands) + manualRoutes.GET(":run_id/:step_run_id", manualHandler.GetPendingCommand) + manualRoutes.PUT(":run_id/:step_run_id", manualHandler.PutContinue) + } +} + +// PlaybookRoutesWithService registers playbook CRUD routes using an injected service. +func PlaybookRoutesWithService(route *gin.Engine, svc orchestrator.PlaybookService) { + log.Trace("Setting up playbook routes") + playbookHandler := playbook_handler.NewPlaybookHandler(svc) + playbookRoutes := route.Group("/playbook") + { + playbookRoutes.GET("/", playbookHandler.GetAllPlaybooks) + playbookRoutes.POST("/", playbookHandler.SubmitPlaybook) + playbookRoutes.GET("/meta/", playbookHandler.GetAllPlaybookMetas) + playbookRoutes.GET("/:id", playbookHandler.GetPlaybookByID) + playbookRoutes.PUT("/:id", playbookHandler.UpdatePlaybookByID) + playbookRoutes.DELETE("/:id", playbookHandler.DeleteByPlaybookID) + } +} + +// ReporterRoutesWithService registers reporter routes using an injected service. +func ReporterRoutesWithService(route *gin.Engine, runner runs.Runner) { + log.Trace("Setting up reporter routes") + reportHandler := reporter_handler.NewReportHandler(runner) + reportRoutes := route.Group("/reporter") + { + reportRoutes.GET("/", reportHandler.GetRuns) + reportRoutes.GET("/:id", reportHandler.GetRunReport) + } +} + +func FinPublic(app *gin.Engine, finHandler *fin_handler.FinHandler) { + log.Trace("Setting up fin protocol routes (registered ahead of the admin auth middleware - see FinPublic doc comment)") + FinPublicRoutes(app, finHandler) +} + +func FinAdmin(app *gin.Engine, finHandler *fin_handler.FinHandler) { + log.Trace("Setting up fin discovery routes") + FinAdminRoutes(app, finHandler) +} + +func FinPublicRoutes(route *gin.Engine, finHandler *fin_handler.FinHandler) { + finRoutes := route.Group("/fin") + { + finRoutes.POST("/register", finHandler.Register) + finAuthenticated := finRoutes.Group("") + finAuthenticated.Use(finHandler.RequireFinToken) + { + finAuthenticated.POST("/poll", finHandler.Poll) + finAuthenticated.PUT("jobs/:job_id", finHandler.SubmitResult) + finAuthenticated.PATCH("jobs/:job_id/status", finHandler.StatusPing) + finAuthenticated.DELETE("/", finHandler.Unregister) + } + } +} + +func FinAdminRoutes(route *gin.Engine, finHandler *fin_handler.FinHandler) { + finRoutes := route.Group("/fin") + { + finRoutes.GET("/", finHandler.List) + finRoutes.GET(":fin_id", finHandler.Get) + finRoutes.DELETE(":fin_id", finHandler.Delete) + } +} diff --git a/pkg/api/status/status_api.go b/internal/transport/http/handlers/status/status_api.go similarity index 96% rename from pkg/api/status/status_api.go rename to internal/transport/http/handlers/status/status_api.go index 622ac654..0cba37b1 100644 --- a/pkg/api/status/status_api.go +++ b/internal/transport/http/handlers/status/status_api.go @@ -3,7 +3,7 @@ package status import ( "net/http" "runtime" - "soarca/pkg/models/api" + "soarca/internal/transport/http/schema" "soarca/pkg/utils" "time" diff --git a/internal/transport/http/handlers/trigger/trigger_api.go b/internal/transport/http/handlers/trigger/trigger_api.go new file mode 100644 index 00000000..ff8307c4 --- /dev/null +++ b/internal/transport/http/handlers/trigger/trigger_api.go @@ -0,0 +1,112 @@ +package trigger + +import ( + "errors" + "fmt" + "io" + "net/http" + "reflect" + "soarca/internal/logger" + "soarca/internal/runs" + apiError "soarca/internal/transport/http/handlers/error" + "soarca/internal/transport/http/schema" + "soarca/pkg/cacao" + "soarca/internal/playbooks/decoder" + + "github.com/gin-gonic/gin" +) + +type Empty struct{} + +var log *logger.Log + +type ITrigger interface { + Execute(context *gin.Context) +} + +func init() { + log = logger.Logger(reflect.TypeOf(Empty{}).PkgPath(), logger.Info, "", logger.Json) +} + +type TriggerHandler struct { + runs runs.Runner +} + +func NewTriggerHandler(runner runs.Runner) *TriggerHandler { + return &TriggerHandler{runs: runner} +} + +func (handler *TriggerHandler) ExecuteById(context *gin.Context) { + log.Trace("received execute by ID") + id := context.Param("id") + var variables cacao.Variables + if context.Request.Body != nil { + jsonData, err := io.ReadAll(context.Request.Body) + if err != nil { + log.Trace("Playbook trigger has failed to decode request body") + apiError.SendErrorResponse(context, http.StatusBadRequest, "Failed to decode request body", "POST /trigger/playbook/"+id, "") + return + } + variables, err = runs.DecodeVariables(jsonData) + if err != nil { + log.Error(err) + apiError.SendErrorResponse(context, http.StatusBadRequest, fmt.Sprintf("Cannot execute. reason: %s", err), "POST /trigger/playbook/"+id, "") + return + } + } + runID, err := handler.runs.StartByID(context.Request.Context(), id, variables) + if err != nil { + log.Error(err) + var validationErr runs.ValidationError + if errors.As(err, &validationErr) { + apiError.SendErrorResponse(context, http.StatusBadRequest, fmt.Sprintf("Cannot execute. reason: %s", validationErr.Error()), "POST /trigger/playbook/"+id, "") + return + } + apiError.SendErrorResponse(context, http.StatusRequestTimeout, err.Error(), "POST "+context.Request.URL.Path, "") + return + } + context.JSON(http.StatusOK, api.RunStarted{ + RunId: runID, + PlaybookId: id, + }) +} + +func (handler *TriggerHandler) Execute(context *gin.Context) { + log.Trace("received execute with body") + jsonData, err := io.ReadAll(context.Request.Body) + if err != nil { + log.Error("failed") + apiError.SendErrorResponse(context, http.StatusBadRequest, + "Failed to marshall json on server side", + "POST /trigger/playbook", "") + return + } + playbook := decoder.DecodeValidate(jsonData) + if playbook == nil { + log.Error("Failed to decode playbook") + apiError.SendErrorResponse(context, http.StatusBadRequest, + "Failed to decode playbook", + "POST /trigger/playbook", "") + return + } + + runID, err := handler.runs.Start(context.Request.Context(), playbook, cacao.Variables{}) + if err != nil { + log.Error(err) + var validationErr runs.ValidationError + if errors.As(err, &validationErr) { + apiError.SendErrorResponse(context, http.StatusBadRequest, fmt.Sprintf("Cannot execute. reason: %s", validationErr.Error()), "POST /trigger/playbook", "") + return + } + apiError.SendErrorResponse(context, + http.StatusRequestTimeout, + err.Error(), + "POST "+context.Request.URL.Path, "") + return + } + context.JSON(http.StatusOK, + api.RunStarted{ + RunId: runID, + PlaybookId: playbook.ID, + }) +} diff --git a/pkg/models/api/error.go b/internal/transport/http/schema/error.go similarity index 100% rename from pkg/models/api/error.go rename to internal/transport/http/schema/error.go diff --git a/internal/transport/http/schema/execution.go b/internal/transport/http/schema/execution.go new file mode 100644 index 00000000..f0c3c091 --- /dev/null +++ b/internal/transport/http/schema/execution.go @@ -0,0 +1,9 @@ +package api + +import "github.com/google/uuid" + +// RunStarted is returned when a playbook run has been accepted and started. +type RunStarted struct { + RunId uuid.UUID `json:"run_id" validate:"required" example:"2c855cd6-bbce-402f-a143-3d6eec346c08"` + PlaybookId string `json:"playbook_id" validate:"required" example:"playbook--0cec398c-db69-4f17-bde4-8ecbcc4a8879"` +} diff --git a/internal/transport/http/schema/manual.go b/internal/transport/http/schema/manual.go new file mode 100644 index 00000000..e2223b10 --- /dev/null +++ b/internal/transport/http/schema/manual.go @@ -0,0 +1,35 @@ +package api + +import ( + "soarca/internal/workflow/capability" + "soarca/pkg/cacao" + "soarca/internal/manual/model" +) + +// Object interfaced to users storing info about pending manual commands +// TODO: change to manualcommandinfo +type PendingCommandData struct { + Type string `bson:"type" json:"type" validate:"required" example:"run-status"` // The type of this content + RunId string `bson:"run_id" json:"run_id" validate:"required"` // The id of the run + PlaybookId string `bson:"playbook_id" json:"playbook_id" validate:"required"` // The id of the CACAO playbook executed by the run + StepId string `bson:"step_id" json:"step_id" validate:"required"` // The id of the step executed by the run + StepRunId string `bson:"step_run_id" json:"step_run_id" validate:"required"` // The id of this specific step invocation. Distinguishes concurrent/repeated pending commands that share the same StepId (e.g. overlapping loop iterations) + Commands []ManualCommand `bson:"commands" json:"commands" validate:"required"` // All commands of the step, in order. A manual step is a single unit of work resolved by one response, but may list multiple commands/instructions + Targets []capability.ResolvedTarget `bson:"targets" json:"targets" validate:"required"` // All targets of the step, in order, together with their resolved authentication information (needed by a human operator to perform the step manually) + OutVariables cacao.Variables `bson:"out_args" json:"out_args" validate:"required"` // Map of cacao variables handled in the step out args with current values and definitions +} + +// One command of a (possibly multi-command) manual step +type ManualCommand struct { + Description string `bson:"description" json:"description" validate:"required"` // The description from the workflow step + Command string `bson:"command" json:"command" validate:"required"` // The command for the agent, either plain or base64 + CommandIsBase64 bool `bson:"commandb64,omitempty" json:"commandb64,omitempty"` // Indicates if the command is in b64 +} + +// The object posted on the manual API Continue() payload +type ManualOutArgsUpdatePayload struct { + Type string `bson:"type" json:"type" validate:"required" example:"string"` // The type of this content + ResponseStatus manual.ManualResponseStatus `bson:"response_status" json:"response_status" validate:"required"` // Indicates status of command + + ResponseOutArgs cacao.Variables `bson:"response_out_args" json:"response_out_args" validate:"required"` // Map of cacao variables storing the out args value, handled in the step out args, with current values and definitions +} diff --git a/internal/transport/http/schema/playbook.go b/internal/transport/http/schema/playbook.go new file mode 100644 index 00000000..2474f190 --- /dev/null +++ b/internal/transport/http/schema/playbook.go @@ -0,0 +1,5 @@ +package api + +import "soarca/internal/playbooks" + +type PlaybookMeta = playbooks.Meta diff --git a/pkg/models/api/reporter.go b/internal/transport/http/schema/reporter.go similarity index 58% rename from pkg/models/api/reporter.go rename to internal/transport/http/schema/reporter.go index 424fc4fd..5313868a 100644 --- a/pkg/models/api/reporter.go +++ b/internal/transport/http/schema/reporter.go @@ -3,8 +3,8 @@ package api import ( "errors" "fmt" - "soarca/pkg/models/cacao" - cache_model "soarca/pkg/models/cache" + "soarca/pkg/cacao" + runstate_model "soarca/internal/runs/state" "time" ) @@ -23,35 +23,36 @@ const ( ExceptionConditionError = "exception_condition_error" AwaitUserInput = "await_user_input" - SuccessfullyExecutedText = "%s execution completed successfully" - FailedText = "something went wrong in the execution of this %s" + SuccessfullyExecutedText = "%s run completed successfully" + FailedText = "something went wrong in the run of this %s" OngoingText = "this %s is currently being executed" - ServerSideErrorText = "there was a server-side problem with the execution of this %s" + ServerSideErrorText = "there was a server-side problem with the run of this %s" ClientSideErrorText = "something in the data provided for this %s raised an issue" - TimeoutErrorText = "the execution of this %s timed out" - ExceptionConditionErrorText = "the execution of this %s raised a playbook exception" - AwaitUserInputText = "waiting for users to provide input for the %s execution" + TimeoutErrorText = "the run of this %s timed out" + ExceptionConditionErrorText = "the run of this %s raised a playbook exception" + AwaitUserInputText = "waiting for users to provide input for the %s run" ) -type PlaybookExecutionReport struct { - Name string `bson:"name" json:"name"` - Description string `bson:"description" json:"description"` - Type string `bson:"type" json:"type"` - ExecutionId string `bson:"execution_id" json:"execution_id"` - PlaybookId string `bson:"playbook_id" json:"playbook_id"` - Started time.Time `bson:"started" json:"started"` - Ended time.Time `bson:"ended" json:"ended"` - Status string `bson:"status" json:"status"` - StatusText string `bson:"status_text" json:"status_text"` - StepResults map[string]StepExecutionReport `bson:"step_results" json:"step_results"` - RequestInterval int `bson:"request_interval" json:"request_interval"` +type PlaybookRunReport struct { + Name string `bson:"name" json:"name"` + Description string `bson:"description" json:"description"` + Type string `bson:"type" json:"type"` + RunId string `bson:"run_id" json:"run_id"` + PlaybookId string `bson:"playbook_id" json:"playbook_id"` + Started time.Time `bson:"started" json:"started"` + Ended time.Time `bson:"ended" json:"ended"` + Status string `bson:"status" json:"status"` + StatusText string `bson:"status_text" json:"status_text"` + StepResults map[string]StepRunReport `bson:"step_results" json:"step_results"` + RequestInterval int `bson:"request_interval" json:"request_interval"` } -type StepExecutionReport struct { +type StepRunReport struct { Name string `bson:"name" json:"name"` Description string `bson:"description" json:"description"` - ExecutionId string `bson:"execution_id" json:"execution_id"` + RunId string `bson:"run_id" json:"run_id"` StepId string `bson:"step_id" json:"step_id"` + StepRunId string `bson:"step_run_id" json:"step_run_id"` Started time.Time `bson:"started" json:"started"` Ended time.Time `bson:"ended" json:"ended"` Status string `bson:"status" json:"status"` @@ -59,17 +60,17 @@ type StepExecutionReport struct { ExecutedBy string `bson:"executed_by" json:"executed_by"` CommandsB64 []string `bson:"commands_b64" json:"commands_b64"` Variables map[string]cacao.Variable `bson:"variables" json:"variables"` - AutomatedExecution bool `bson:"automated_execution" json:"automated_execution"` + AutomatedRun bool `bson:"automated_run" json:"automated_run"` // Make sure we can have a playbookID for playbook actions, and also - // the execution ID for the invoked playbook + // the run ID for the invoked playbook } -func CacheStatusEnum2String(status cache_model.Status) string { +func RunStatusEnum2String(status runstate_model.Status) string { return status.String() } // Level must be either "step" or "playbook" -func GetCacheStatusText(status string, level string) (string, error) { +func GetRunStatusText(status string, level string) (string, error) { if level != ReportLevelPlaybook && level != ReportLevelStep { return "", errors.New("invalid reporting level provided. use either 'playbook' or 'step'") } @@ -91,6 +92,6 @@ func GetCacheStatusText(status string, level string) (string, error) { case AwaitUserInput: return fmt.Sprintf(AwaitUserInputText, level), nil default: - return "", errors.New("unable to read execution information status") + return "", errors.New("unable to read run information status") } } diff --git a/pkg/models/api/status.go b/internal/transport/http/schema/status.go similarity index 100% rename from pkg/models/api/status.go rename to internal/transport/http/schema/status.go diff --git a/internal/transport/http/server.go b/internal/transport/http/server.go new file mode 100644 index 00000000..0aa4360d --- /dev/null +++ b/internal/transport/http/server.go @@ -0,0 +1,130 @@ +package httptransport + +import ( + "fmt" + "os" + "reflect" + "strings" + + "soarca/internal/config" + "soarca/internal/logger" + orchestrator "soarca/internal/orchestrator" + "soarca/internal/transport/http/handlers" + finapi "soarca/internal/transport/http/handlers/fin" + + "github.com/COSSAS/gauth" + "github.com/gin-gonic/gin" +) + +var log *logger.Log + +type Empty struct{} + +func init() { + log = logger.Logger(reflect.TypeOf(Empty{}).PkgPath(), logger.Info, "", logger.Json) +} + +// Options is the configuration the HTTP transport needs. It is deliberately +// narrower than the application config: anything the orchestrator owns +// (storage, TheHive, outbound TLS) does not belong here. +type Options struct { + Server config.ServerConfig + Fin config.FinConfig + Auth config.AuthConfig + CORS config.CORSConfig +} + +// Server owns the HTTP transport wiring: route registration, middleware and +// listener startup. It holds the orchestrator's use case surface and nothing else. +type Server struct { + ops orchestrator.Operations + opts Options + finHandler *finapi.FinHandler +} + +// New creates an HTTP server adapter over the given operations. +func New(ops orchestrator.Operations, opts Options) *Server { + finHandler := finapi.NewFinHandler( + ops.Fins, + ops.Work, + finapi.Config{ + RegistrationToken: opts.Fin.RegistrationToken, + PollIntervalSeconds: opts.Fin.PollIntervalSeconds, + LongPollTimeoutSeconds: opts.Fin.LongPollTimeoutSeconds, + JobLeaseSeconds: opts.Fin.JobLeaseSeconds, + StaleAfter: opts.Fin.StaleAfter, + }, + ) + + return &Server{ops: ops, opts: opts, finHandler: finHandler} +} + +// SetupServer initializes the Gin engine with all routes and middleware. +func (s *Server) SetupServer() (*gin.Engine, error) { + engine := gin.New() + + log.Info("Log level is info") + log.Debug("Log level is debug") + log.Trace("Log level is trace") + + origins := strings.Split(strings.ReplaceAll(s.opts.CORS.AllowedOrigins, " ", ""), ",") + api.Cors(engine, origins) + + api.FinPublic(engine, s.finHandler) + + if err := s.setupAuthMiddleware(engine); err != nil { + return nil, fmt.Errorf("failed to setup auth middleware: %w", err) + } + + api.TriggerRoutes(engine, api.NewTriggerHandler(s.ops.Runs)) + api.StatusRoutes(engine) + api.PlaybookRoutesWithService(engine, s.ops.Playbooks) + api.ReporterRoutesWithService(engine, s.ops.Runs) + api.ManualRoutes(engine, api.NewManualHandler(s.ops.Manual)) + api.FinAdmin(engine, s.finHandler) + api.Logging(engine) + api.Swagger(engine) + + return engine, nil +} + +// setupAuthMiddleware configures authentication if enabled. +func (s *Server) setupAuthMiddleware(engine *gin.Engine) error { + if !s.opts.Auth.Enabled { + return nil + } + + log.Info("Enabling authentication middleware") + auth, err := gauth.New(gauth.DefaultConfig()) + if err != nil { + return fmt.Errorf("failed to initialize authenticator: %w", err) + } + engine.Use(auth.LoadAuthContext()) + engine.Use(auth.Middleware([]string{"soarca_admin"})) + return nil +} + +// RunServer starts the HTTP server on the configured port. +func (s *Server) RunServer(engine *gin.Engine) error { + if s.opts.Server.EnableTLS { + if err := validateCertificates(s.opts.Server.CertFile, s.opts.Server.CertKey); err != nil { + return err + } + log.Infof("Starting HTTPS server on port %s", s.opts.Server.Port) + return engine.RunTLS(":"+s.opts.Server.Port, s.opts.Server.CertFile, s.opts.Server.CertKey) + } + + log.Infof("Starting HTTP server on port %s", s.opts.Server.Port) + return engine.Run(":" + s.opts.Server.Port) +} + +// validateCertificates checks that TLS certificate files exist. +func validateCertificates(certFile, keyFile string) error { + if _, err := os.Stat(certFile); os.IsNotExist(err) { + return fmt.Errorf("certificate file not found: %s", certFile) + } + if _, err := os.Stat(keyFile); os.IsNotExist(err) { + return fmt.Errorf("key file not found: %s", keyFile) + } + return nil +} diff --git a/internal/transport/http/server_test.go b/internal/transport/http/server_test.go new file mode 100644 index 00000000..66ff4fcb --- /dev/null +++ b/internal/transport/http/server_test.go @@ -0,0 +1,71 @@ +package httptransport + +import ( + "os" + "path/filepath" + "testing" + + "soarca/internal/config" + orchestrator "soarca/internal/orchestrator" +) + +func testOperations(t *testing.T) orchestrator.Operations { + t.Helper() + + app, err := orchestrator.New(orchestrator.Options{ + Storage: config.StorageConfig{DatabaseURL: "sqlite://:memory:"}, + RunState: config.RunStateConfig{MaxRuns: 2}, + }) + if err != nil { + t.Fatalf("orchestrator.New() returned error: %v", err) + } + t.Cleanup(func() { + if err := app.Close(); err != nil { + t.Fatalf("orchestrator.Close() returned error: %v", err) + } + }) + return app.Operations() +} + +func TestSetupServerReturnsEngineWithAuthDisabled(t *testing.T) { + opts := Options{ + Server: config.ServerConfig{Port: "0"}, + Fin: config.FinConfig{}, + Auth: config.AuthConfig{Enabled: false}, + CORS: config.CORSConfig{AllowedOrigins: "*"}, + } + + server := New(testOperations(t), opts) + + engine, err := server.SetupServer() + if err != nil { + t.Fatalf("SetupServer() returned error: %v", err) + } + if engine == nil { + t.Fatal("SetupServer() returned nil engine") + } + if got := len(engine.Routes()); got == 0 { + t.Fatal("expected routes to be registered") + } +} + +func TestValidateCertificates(t *testing.T) { + dir := t.TempDir() + certFile := filepath.Join(dir, "server.crt") + keyFile := filepath.Join(dir, "server.key") + + if err := os.WriteFile(certFile, []byte("cert"), 0o600); err != nil { + t.Fatalf("failed to write cert file: %v", err) + } + if err := os.WriteFile(keyFile, []byte("key"), 0o600); err != nil { + t.Fatalf("failed to write key file: %v", err) + } + + if err := validateCertificates(certFile, keyFile); err != nil { + t.Fatalf("validateCertificates() returned error: %v", err) + } + + if err := validateCertificates(filepath.Join(dir, "missing.crt"), keyFile); err == nil { + t.Fatal("expected missing certificate file to fail") + } +} diff --git a/internal/workflow/capability/contract.go b/internal/workflow/capability/contract.go new file mode 100644 index 00000000..04d40c45 --- /dev/null +++ b/internal/workflow/capability/contract.go @@ -0,0 +1,45 @@ +package capability + +import ( + "soarca/pkg/cacao" + "soarca/internal/runs/model" +) + +// ResolvedTarget pairs a step target with the authentication information +// resolved for it. Each target in a step may legitimately use different +// authentication, so auth is nested per-target rather than a single +// top-level field. +// +// This is also SOARCA's one canonical wire shape for "a target plus its +// resolved auth" wherever that needs to cross an API boundary (the Manual +// API's pending-command payload, the Fin protocol's job payload) — reused +// directly rather than each protocol defining its own equivalent-but- +// differently-shaped DTO. +type ResolvedTarget struct { + Target cacao.AgentTarget `bson:"target" json:"target" validate:"required"` + Authentication cacao.AuthenticationInformation `bson:"authentication,omitempty" json:"authentication,omitempty"` +} + +// Context carries everything a capability needs to execute a single step. +// Commands and Targets are both plain arrays (0, 1, or many) — a capability +// receives the full step in one call and is responsible for iterating over +// its own commands/targets (and any target-level parallelism) internally, +// rather than being invoked once per (command, target) pair. +type Context struct { + Commands []cacao.Command + Targets []ResolvedTarget + Step cacao.Step + Variables cacao.Variables + // Agent is the step's resolved agent_definitions entry. Built-in + // capabilities don't need it (they're already selected by + // agent.Type), but the Fin fallback capability does: it has no single + // static type of its own, so it reads Agent.Type here to route the + // job to the right pool of registered Fins. + Agent cacao.AgentTarget +} + +type ICapability interface { + Execute(metadata run.Metadata, + context Context) (cacao.Variables, error) + GetType() string +} diff --git a/internal/workflow/capability/fin/capability.go b/internal/workflow/capability/fin/capability.go new file mode 100644 index 00000000..2757b216 --- /dev/null +++ b/internal/workflow/capability/fin/capability.go @@ -0,0 +1,157 @@ +package fin + +import ( + "context" + "errors" + "reflect" + "time" + + "soarca/internal/logger" + "soarca/internal/store" + "soarca/internal/workflow/capability" + "soarca/pkg/cacao" + "soarca/pkg/fins/protocol" + "soarca/internal/runs/model" + "soarca/pkg/utils" + timeUtil "soarca/pkg/utils/time" + + "soarca/pkg/utils/guid" +) + +type Empty struct{} + +var log *logger.Log + +func init() { + log = logger.Logger(reflect.TypeOf(Empty{}).PkgPath(), logger.Info, "", logger.Json) +} + +type IJobQueue interface { + Enqueue(ctx context.Context, job fin.Job) (fin.JobResult, error) +} + +type Capability struct { + queue IJobQueue + guid guid.IGuid + store storage.FinStore + time timeUtil.ITime + staleAfter time.Duration +} + +// Dependencies groups the dependencies needed to construct a Fin capability. +type Dependencies struct { + Queue IJobQueue + GUID guid.IGuid + Store storage.FinStore + Time timeUtil.ITime + StaleAfter time.Duration +} + +func New(deps Dependencies) *Capability { + return &Capability{ + queue: deps.Queue, + guid: deps.GUID, + store: deps.Store, + time: deps.Time, + staleAfter: deps.StaleAfter, + } +} + +func (finCapability *Capability) GetType() string { return "" } + +func (finCapability *Capability) Execute(metadata run.Metadata, commandContext capability.Context) (cacao.Variables, error) { + log.Trace(metadata.RunId) + if err := finCapability.checkCapableFin(commandContext.Agent.Type); err != nil { + log.Error(err) + return cacao.NewVariables(), err + } + timeout := leaseDuration(commandContext.Step.Timeout) + job := fin.Job{ + JobId: finCapability.guid.New(), + RunId: metadata.RunId, + PlaybookId: metadata.PlaybookId, + StepId: metadata.StepId, + StepRunId: metadata.StepRunId, + CapabilityType: commandContext.Agent.Type, + LeaseExpiresInSeconds: int(timeout.Seconds()), + Step: fin.StepInfo{Name: commandContext.Step.Name, Description: commandContext.Step.Description, Timeout: commandContext.Step.Timeout, Delay: commandContext.Step.Delay}, + Commands: toFinCommands(commandContext.Commands), + Targets: commandContext.Targets, + Variables: commandContext.Variables, + } + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + result, err := finCapability.queue.Enqueue(ctx, job) + if err != nil { + log.Error("fin job ", job.JobId.String(), " did not complete: ", err) + return cacao.NewVariables(), err + } + if result.State == fin.JobStateFailure { + resultErr := errors.New(result.Error) + if result.Error == "" { + resultErr = errors.New("fin job reported failure without an error message") + } + log.Error(resultErr) + return result.Variables, resultErr + } + return result.Variables, nil +} + +func (finCapability *Capability) checkCapableFin(capabilityType string) error { + if finCapability.store == nil { + return nil + } + records, err := finCapability.store.List(context.Background()) + if err != nil { + log.Warning("failed to list registered fins for capability type ", capabilityType, ": ", err) + return nil + } + now := finCapability.time.Now() + matched := false + live := false + staleFinIds := []string{} + for _, record := range records { + if !hasCapability(record, capabilityType) { + continue + } + matched = true + if now.Sub(record.LastSeen) <= finCapability.staleAfter { + live = true + continue + } + staleFinIds = append(staleFinIds, record.FinId) + } + if !matched { + return fin.ErrNoCapableFin{CapabilityType: capabilityType} + } + if !live { + return fin.ErrOnlyStaleCapableFins{CapabilityType: capabilityType, FinIds: staleFinIds, StaleAfter: finCapability.staleAfter} + } + return nil +} + +func hasCapability(record fin.Record, capabilityType string) bool { + for _, cap := range record.Capabilities { + if cap.Type == capabilityType { + return true + } + } + return false +} + +func leaseDuration(stepTimeoutMillis int) time.Duration { + if stepTimeoutMillis <= 0 { + fallback := utils.DefaultStepTimeout() + log.Warning("timeout is not set or set to 0, fallback timeout of ", fallback, " is used to complete step") + return fallback + } + return time.Duration(stepTimeoutMillis) * time.Millisecond +} + +func toFinCommands(commands []cacao.Command) []fin.Command { + converted := make([]fin.Command, 0, len(commands)) + for _, command := range commands { + converted = append(converted, fin.Command{Type: command.Type, Command: command.Command}) + } + return converted +} diff --git a/internal/workflow/capability/fin/fin_test.go b/internal/workflow/capability/fin/fin_test.go new file mode 100644 index 00000000..bbf98263 --- /dev/null +++ b/internal/workflow/capability/fin/fin_test.go @@ -0,0 +1,317 @@ +package fin + +import ( + "context" + "errors" + "testing" + "time" + + "soarca/internal/workflow/capability" + "soarca/pkg/cacao" + "soarca/pkg/fins/protocol" + "soarca/internal/runs/model" + "soarca/pkg/utils" + "soarca/test/unittest/mocks/mock_guid" + mock_time "soarca/test/unittest/mocks/mock_utils/time" + + "github.com/go-playground/assert/v2" + "github.com/google/uuid" + "github.com/stretchr/testify/mock" +) + +type mockQueue struct { + mock.Mock +} + +func (m *mockQueue) Enqueue(ctx context.Context, job fin.Job) (fin.JobResult, error) { + args := m.Called(ctx, job) + return args.Get(0).(fin.JobResult), args.Error(1) +} + +type mockStore struct { + mock.Mock +} + +func (m *mockStore) Create(ctx context.Context, record fin.Record) error { + args := m.Called(ctx, record) + return args.Error(0) +} + +func (m *mockStore) Get(ctx context.Context, finID string) (fin.Record, error) { + args := m.Called(ctx, finID) + return args.Get(0).(fin.Record), args.Error(1) +} + +func (m *mockStore) GetByTokenHash(ctx context.Context, tokenHash string) (fin.Record, error) { + args := m.Called(ctx, tokenHash) + return args.Get(0).(fin.Record), args.Error(1) +} + +func (m *mockStore) List(ctx context.Context) ([]fin.Record, error) { + args := m.Called(ctx) + return args.Get(0).([]fin.Record), args.Error(1) +} + +func (m *mockStore) Touch(ctx context.Context, finID string, lastSeen time.Time) error { + args := m.Called(ctx, finID, lastSeen) + return args.Error(0) +} + +func (m *mockStore) Delete(ctx context.Context, finID string) error { + args := m.Called(ctx, finID) + return args.Error(0) +} + +func newMetadataAndContext() (run.Metadata, capability.Context) { + metadata := run.Metadata{ + RunId: uuid.New(), + PlaybookId: "playbook--test", + StepId: "action--test", + StepRunId: uuid.New(), + } + commandContext := capability.Context{ + Commands: []cacao.Command{{Type: "http-api", Command: "GET /"}}, + Targets: []capability.ResolvedTarget{{ + Target: cacao.AgentTarget{Type: "target", Name: "myself"}, + Authentication: cacao.AuthenticationInformation{Type: "user-auth", Username: "operator"}, + }}, + Step: cacao.Step{Type: cacao.StepTypeAction, Name: "test step", Timeout: 5000}, + Variables: cacao.NewVariables(), + Agent: cacao.AgentTarget{Type: "http-executor", Name: "some fin"}, + } + return metadata, commandContext +} + +func TestExecuteEnqueuesJobRoutedByAgentTypeAndReturnsResultVariables(t *testing.T) { + queue := new(mockQueue) + guidMock := new(mock_guid.Mock_Guid) + jobId := uuid.New() + guidMock.On("New").Return(jobId) + + metadata, commandContext := newMetadataAndContext() + + expectedVariables := cacao.NewVariables(cacao.Variable{Type: cacao.VariableTypeString, Name: "__out__", Value: "ok"}) + + queue.On("Enqueue", mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + job := args.Get(1).(fin.Job) + assert.Equal(t, job.JobId, jobId) + assert.Equal(t, job.RunId, metadata.RunId) + assert.Equal(t, job.PlaybookId, metadata.PlaybookId) + assert.Equal(t, job.StepId, metadata.StepId) + assert.Equal(t, job.StepRunId, metadata.StepRunId) + // The job must be routed by the step's resolved agent.Type, not + // by any built-in capability name - this is the whole point of + // FinCapability being a dynamic-type fallback. + assert.Equal(t, job.CapabilityType, "http-executor") + assert.Equal(t, job.LeaseExpiresInSeconds, 5) + assert.Equal(t, len(job.Commands), 1) + assert.Equal(t, job.Commands[0].Command, "GET /") + assert.Equal(t, len(job.Targets), 1) + assert.Equal(t, job.Targets[0].Target.Name, "myself") + assert.Equal(t, job.Targets[0].Authentication.Username, "operator") + }). + Return(fin.JobResult{State: fin.JobStateSuccess, Variables: expectedVariables}, nil) + + // A nil store disables the liveness check entirely, so Execute + // always proceeds straight to Enqueue - this is the pre-existing + // behavior these tests are pinning. + finCapability := New(Dependencies{Queue: queue, GUID: guidMock}) + variables, err := finCapability.Execute(metadata, commandContext) + + assert.Equal(t, err, nil) + assert.Equal(t, variables, expectedVariables) + queue.AssertExpectations(t) +} + +func TestExecuteReturnsErrorWhenJobResultIsFailure(t *testing.T) { + queue := new(mockQueue) + guidMock := new(mock_guid.Mock_Guid) + guidMock.On("New").Return(uuid.New()) + + metadata, commandContext := newMetadataAndContext() + + queue.On("Enqueue", mock.Anything, mock.Anything). + Return(fin.JobResult{State: fin.JobStateFailure, Error: "command exited 1"}, nil) + + finCapability := New(Dependencies{Queue: queue, GUID: guidMock}) + _, err := finCapability.Execute(metadata, commandContext) + + assert.NotEqual(t, err, nil) + assert.Equal(t, err.Error(), "command exited 1") +} + +func TestExecuteReturnsErrorWhenQueueFailsOrTimesOut(t *testing.T) { + queue := new(mockQueue) + guidMock := new(mock_guid.Mock_Guid) + guidMock.On("New").Return(uuid.New()) + + metadata, commandContext := newMetadataAndContext() + + queueErr := errors.New("context deadline exceeded") + queue.On("Enqueue", mock.Anything, mock.Anything). + Return(fin.JobResult{}, queueErr) + + finCapability := New(Dependencies{Queue: queue, GUID: guidMock}) + _, err := finCapability.Execute(metadata, commandContext) + + assert.Equal(t, err, queueErr) +} + +func TestExecuteFallsBackToDefaultLeaseWhenStepTimeoutIsUnset(t *testing.T) { + queue := new(mockQueue) + guidMock := new(mock_guid.Mock_Guid) + guidMock.On("New").Return(uuid.New()) + + metadata, commandContext := newMetadataAndContext() + commandContext.Step.Timeout = 0 + + queue.On("Enqueue", mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + job := args.Get(1).(fin.Job) + assert.Equal(t, job.LeaseExpiresInSeconds, int(utils.DefaultStepTimeout().Seconds())) + }). + Return(fin.JobResult{State: fin.JobStateSuccess, Variables: cacao.NewVariables()}, nil) + + finCapability := New(Dependencies{Queue: queue, GUID: guidMock}) + _, err := finCapability.Execute(metadata, commandContext) + assert.Equal(t, err, nil) +} + +// ############################################################################ +// Fail-fast liveness check (checkCapableFin) +// ############################################################################ + +func TestExecuteFailsFastWhenNoFinIsRegisteredForCapabilityType(t *testing.T) { + queue := new(mockQueue) + guidMock := new(mock_guid.Mock_Guid) + store := new(mockStore) + clock := new(mock_time.MockTime) + clock.On("Now").Return(time.Unix(1000, 0)) + + metadata, commandContext := newMetadataAndContext() + + store.On("List", mock.Anything).Return([]fin.Record{ + {FinId: "other-fin", LastSeen: time.Unix(1000, 0), Capabilities: []fin.Capability{{Type: "some-other-type"}}}, + }, nil) + + finCapability := New(Dependencies{ + Queue: queue, + GUID: guidMock, + Store: store, + Time: clock, + StaleAfter: time.Minute, + }) + _, err := finCapability.Execute(metadata, commandContext) + + var noCapableFin fin.ErrNoCapableFin + assert.Equal(t, errors.As(err, &noCapableFin), true) + assert.Equal(t, noCapableFin.CapabilityType, "http-executor") + // Must never even touch the queue - that is the whole point of failing fast. + queue.AssertNotCalled(t, "Enqueue", mock.Anything, mock.Anything) +} + +func TestExecuteFailsFastWhenEveryCapableFinIsStale(t *testing.T) { + queue := new(mockQueue) + guidMock := new(mock_guid.Mock_Guid) + store := new(mockStore) + clock := new(mock_time.MockTime) + now := time.Unix(10000, 0) + clock.On("Now").Return(now) + + metadata, commandContext := newMetadataAndContext() + + staleAfter := time.Minute + store.On("List", mock.Anything).Return([]fin.Record{ + { + FinId: "stale-fin", + LastSeen: now.Add(-2 * staleAfter), + Capabilities: []fin.Capability{{Type: "http-executor"}}, + }, + }, nil) + + finCapability := New(Dependencies{ + Queue: queue, + GUID: guidMock, + Store: store, + Time: clock, + StaleAfter: staleAfter, + }) + _, err := finCapability.Execute(metadata, commandContext) + + var onlyStale fin.ErrOnlyStaleCapableFins + assert.Equal(t, errors.As(err, &onlyStale), true) + assert.Equal(t, onlyStale.CapabilityType, "http-executor") + assert.Equal(t, onlyStale.FinIds, []string{"stale-fin"}) + queue.AssertNotCalled(t, "Enqueue", mock.Anything, mock.Anything) +} + +func TestExecuteProceedsWhenAtLeastOneCapableFinIsLive(t *testing.T) { + queue := new(mockQueue) + guidMock := new(mock_guid.Mock_Guid) + guidMock.On("New").Return(uuid.New()) + store := new(mockStore) + clock := new(mock_time.MockTime) + now := time.Unix(10000, 0) + clock.On("Now").Return(now) + + metadata, commandContext := newMetadataAndContext() + + staleAfter := time.Minute + store.On("List", mock.Anything).Return([]fin.Record{ + { + FinId: "stale-fin", + LastSeen: now.Add(-2 * staleAfter), + Capabilities: []fin.Capability{{Type: "http-executor"}}, + }, + { + FinId: "live-fin", + LastSeen: now.Add(-1 * time.Second), + Capabilities: []fin.Capability{{Type: "http-executor"}}, + }, + }, nil) + + queue.On("Enqueue", mock.Anything, mock.Anything). + Return(fin.JobResult{State: fin.JobStateSuccess, Variables: cacao.NewVariables()}, nil) + + finCapability := New(Dependencies{ + Queue: queue, + GUID: guidMock, + Store: store, + Time: clock, + StaleAfter: staleAfter, + }) + _, err := finCapability.Execute(metadata, commandContext) + + assert.Equal(t, err, nil) + queue.AssertExpectations(t) +} + +func TestExecuteProceedsWhenStoreListFails(t *testing.T) { + queue := new(mockQueue) + guidMock := new(mock_guid.Mock_Guid) + guidMock.On("New").Return(uuid.New()) + store := new(mockStore) + clock := new(mock_time.MockTime) + + metadata, commandContext := newMetadataAndContext() + + store.On("List", mock.Anything).Return([]fin.Record{}, errors.New("database unavailable")) + queue.On("Enqueue", mock.Anything, mock.Anything). + Return(fin.JobResult{State: fin.JobStateSuccess, Variables: cacao.NewVariables()}, nil) + + // A store error must fail open - fall back to the pre-existing + // enqueue-and-wait behavior rather than blocking a step over an + // inability to check liveness. + finCapability := New(Dependencies{ + Queue: queue, + GUID: guidMock, + Store: store, + Time: clock, + StaleAfter: time.Minute, + }) + _, err := finCapability.Execute(metadata, commandContext) + + assert.Equal(t, err, nil) + queue.AssertExpectations(t) +} diff --git a/internal/workflow/capability/fin/queue/jobs.go b/internal/workflow/capability/fin/queue/jobs.go new file mode 100644 index 00000000..2b49e480 --- /dev/null +++ b/internal/workflow/capability/fin/queue/jobs.go @@ -0,0 +1,258 @@ +// Package queue implements the in-memory Fin job queue. +// +// It is intentionally not persisted because in-flight runs are not +// resumed across restarts either. +package queue + +import ( + "context" + "reflect" + "sync" + "time" + + "soarca/internal/logger" + "soarca/pkg/fins/protocol" + + "github.com/google/uuid" +) + +type Empty struct{} + +var component = reflect.TypeOf(Empty{}).PkgPath() +var log *logger.Log + +func init() { + log = logger.Logger(component, logger.Info, "", logger.Json) +} + +// defaultLeaseDuration is used when a Job does not specify a positive +// LeaseExpiresInSeconds. +const defaultLeaseDuration = 60 * time.Second + +// defaultSweepInterval is how often expired leases are checked and +// requeued. +const defaultSweepInterval = time.Second + +// entry is the queue's internal bookkeeping for one job: the job payload +// itself, the channel its Enqueue caller is blocked reading from, and its +// current lease state (unleased jobs have an empty LeasedTo). +type entry struct { + job fin.Job + resultCh chan fin.JobResult + leasedTo string + leaseExpiry time.Time +} + +// Queue is the in-memory Fin job queue: a set of per-CapabilityType FIFO +// queues of unleased jobs, plus a map of currently-leased jobs keyed by +// JobId. Safe for concurrent use. +type Queue struct { + mu sync.Mutex + pending map[string][]*entry // capability type -> FIFO of unleased jobs + leased map[uuid.UUID]*entry // job id -> currently-leased job + notify chan struct{} // closed and replaced whenever queue state changes, to wake blocked Claim callers + stop chan struct{} + done chan struct{} +} + +// New creates an empty Queue and starts its background lease-expiry +// sweeper. Call Close to stop the sweeper goroutine (e.g. on shutdown, or +// in tests). +func New() *Queue { + q := &Queue{ + pending: map[string][]*entry{}, + leased: map[uuid.UUID]*entry{}, + notify: make(chan struct{}), + stop: make(chan struct{}), + done: make(chan struct{}), + } + go q.sweepLoop() + return q +} + +// Close stops the background lease-expiry sweeper. It does not affect +// already-enqueued or leased jobs. +func (q *Queue) Close() { + close(q.stop) + <-q.done +} + +// Enqueue registers job under its CapabilityType and blocks until a +// claiming Fin submits a result, or ctx is done - whichever comes first. If +// ctx is done first, job is removed from the queue (wherever it currently +// is: still pending, or leased but not yet resolved) so a late claim/result +// can no longer observe or resolve it. +func (q *Queue) Enqueue(ctx context.Context, job fin.Job) (fin.JobResult, error) { + e := &entry{job: job, resultCh: make(chan fin.JobResult, 1)} + + q.mu.Lock() + q.pending[job.CapabilityType] = append(q.pending[job.CapabilityType], e) + q.broadcastLocked() + q.mu.Unlock() + + log.Trace("enqueued job ", job.JobId.String(), " for capability type ", job.CapabilityType) + + select { + case result := <-e.resultCh: + return result, nil + case <-ctx.Done(): + q.removeJob(job.JobId, job.CapabilityType) + return fin.JobResult{}, ctx.Err() + } +} + +// Claim blocks (long-poll) until a job is available under one of +// capabilityTypes, or ctx is done - whichever comes first. On success, the +// returned job is leased to finId until its lease expires or a result is +// submitted for it. +func (q *Queue) Claim(ctx context.Context, capabilityTypes []string, finId string) (fin.Job, error) { + for { + q.mu.Lock() + for _, capabilityType := range capabilityTypes { + jobs := q.pending[capabilityType] + if len(jobs) == 0 { + continue + } + e := jobs[0] + q.pending[capabilityType] = jobs[1:] + if len(q.pending[capabilityType]) == 0 { + delete(q.pending, capabilityType) + } + e.leasedTo = finId + e.leaseExpiry = time.Now().Add(leaseDuration(e.job)) + q.leased[e.job.JobId] = e + q.mu.Unlock() + log.Trace("claimed job ", e.job.JobId.String(), " for fin ", finId) + return e.job, nil + } + waitCh := q.notify + q.mu.Unlock() + + select { + case <-waitCh: + continue + case <-ctx.Done(): + return fin.Job{}, ctx.Err() + } + } +} + +// Submit delivers result for jobId to the Enqueue caller waiting on it, +// provided jobId is currently leased to finId. If the Enqueue caller has +// already given up (ctx done), result is dropped silently - there is +// nothing left to deliver it to. +func (q *Queue) Submit(jobId uuid.UUID, finId string, result fin.JobResult) error { + q.mu.Lock() + e, ok := q.leased[jobId] + if !ok { + q.mu.Unlock() + return fin.ErrJobNotFound{JobId: jobId.String()} + } + if e.leasedTo != finId { + q.mu.Unlock() + return fin.ErrJobNotLeasedToFin{JobId: jobId.String(), FinId: finId} + } + delete(q.leased, jobId) + q.mu.Unlock() + + select { + case e.resultCh <- result: + default: + log.Warning("result submitted for job ", jobId.String(), " but nothing is waiting for it anymore (deadline already exceeded); dropping") + } + return nil +} + +// ExtendLease refreshes a leased job's timeout for status-ping-driven work. +func (q *Queue) ExtendLease(jobId uuid.UUID, finId string, extendBySeconds int) error { + q.mu.Lock() + defer q.mu.Unlock() + + e, ok := q.leased[jobId] + if !ok { + return fin.ErrJobNotFound{JobId: jobId.String()} + } + if e.leasedTo != finId { + return fin.ErrJobNotLeasedToFin{JobId: jobId.String(), FinId: finId} + } + duration := time.Duration(extendBySeconds) * time.Second + if extendBySeconds <= 0 { + duration = defaultLeaseDuration + } + e.leaseExpiry = time.Now().Add(duration) + return nil +} + +// removeJob deletes jobId from wherever it currently sits (pending or +// leased). Used when an Enqueue caller's ctx is done before a result +// arrives. +func (q *Queue) removeJob(jobId uuid.UUID, capabilityType string) { + q.mu.Lock() + defer q.mu.Unlock() + + if jobs, ok := q.pending[capabilityType]; ok { + for i, e := range jobs { + if e.job.JobId == jobId { + q.pending[capabilityType] = append(jobs[:i], jobs[i+1:]...) + if len(q.pending[capabilityType]) == 0 { + delete(q.pending, capabilityType) + } + break + } + } + } + delete(q.leased, jobId) +} + +// sweepLoop periodically requeues jobs whose lease has expired without a +// result or status ping, so another Fin registered under the same +// CapabilityType can claim them. +func (q *Queue) sweepLoop() { + defer close(q.done) + ticker := time.NewTicker(defaultSweepInterval) + defer ticker.Stop() + + for { + select { + case <-q.stop: + return + case <-ticker.C: + q.sweepExpiredLeases() + } + } +} + +func (q *Queue) sweepExpiredLeases() { + q.mu.Lock() + defer q.mu.Unlock() + + now := time.Now() + requeued := false + for jobId, e := range q.leased { + if now.Before(e.leaseExpiry) { + continue + } + log.Warning("lease expired for job ", jobId.String(), " (was leased to fin ", e.leasedTo, "); requeuing for capability type ", e.job.CapabilityType) + delete(q.leased, jobId) + e.leasedTo = "" + q.pending[e.job.CapabilityType] = append(q.pending[e.job.CapabilityType], e) + requeued = true + } + if requeued { + q.broadcastLocked() + } +} + +// broadcastLocked wakes every Claim call currently blocked waiting for new +// work. Must be called with q.mu held. +func (q *Queue) broadcastLocked() { + close(q.notify) + q.notify = make(chan struct{}) +} + +func leaseDuration(job fin.Job) time.Duration { + if job.LeaseExpiresInSeconds <= 0 { + return defaultLeaseDuration + } + return time.Duration(job.LeaseExpiresInSeconds) * time.Second +} diff --git a/internal/workflow/capability/fin/queue/queue_test.go b/internal/workflow/capability/fin/queue/queue_test.go new file mode 100644 index 00000000..5d301623 --- /dev/null +++ b/internal/workflow/capability/fin/queue/queue_test.go @@ -0,0 +1,290 @@ +package queue + +import ( + "context" + "sync" + "testing" + "time" + + "soarca/pkg/fins/protocol" + + "github.com/go-playground/assert/v2" + "github.com/google/uuid" +) + +func newJob(capabilityType string) fin.Job { + return fin.Job{ + JobId: uuid.New(), + RunId: uuid.New(), + StepId: "step--1", + StepRunId: uuid.New(), + CapabilityType: capabilityType, + LeaseExpiresInSeconds: 60, + } +} + +func TestEnqueueClaimSubmitRoundTrip(t *testing.T) { + q := New() + defer q.Close() + + job := newJob("pong") + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + var wg sync.WaitGroup + wg.Add(1) + var result fin.JobResult + var enqueueErr error + go func() { + defer wg.Done() + result, enqueueErr = q.Enqueue(ctx, job) + }() + + claimed, err := q.Claim(ctx, []string{"pong"}, "fin-1") + if err != nil { + t.Fatal(err) + } + assert.Equal(t, claimed.JobId, job.JobId) + + err = q.Submit(job.JobId, "fin-1", fin.JobResult{State: fin.JobStateSuccess}) + if err != nil { + t.Fatal(err) + } + + wg.Wait() + if enqueueErr != nil { + t.Fatal(enqueueErr) + } + assert.Equal(t, result.State, fin.JobStateSuccess) +} + +func TestClaimBlocksUntilJobIsEnqueued(t *testing.T) { + q := New() + defer q.Close() + + claimCtx, claimCancel := context.WithTimeout(context.Background(), 2*time.Second) + defer claimCancel() + + claimed := make(chan fin.Job, 1) + go func() { + job, err := q.Claim(claimCtx, []string{"pong"}, "fin-1") + if err == nil { + claimed <- job + } + }() + + // Give the Claim goroutine time to start blocking before anything is + // enqueued, to actually exercise the long-poll wait path. + time.Sleep(50 * time.Millisecond) + + job := newJob("pong") + enqueueCtx, enqueueCancel := context.WithTimeout(context.Background(), 2*time.Second) + defer enqueueCancel() + go func() { _, _ = q.Enqueue(enqueueCtx, job) }() + + select { + case got := <-claimed: + assert.Equal(t, got.JobId, job.JobId) + case <-time.After(2 * time.Second): + t.Fatal("Claim never returned the job enqueued while it was blocked") + } +} + +func TestClaimOnlyMatchesRequestedCapabilityTypes(t *testing.T) { + q := New() + defer q.Close() + + pingJob := newJob("ping") + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + go func() { _, _ = q.Enqueue(ctx, pingJob) }() + + time.Sleep(20 * time.Millisecond) + + claimCtx, claimCancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer claimCancel() + _, err := q.Claim(claimCtx, []string{"pong"}, "fin-1") + if err == nil { + t.Fatal("expected Claim to time out: no job of a matching capability type is queued") + } +} + +func TestEnqueueContextDoneRemovesUnclaimedJob(t *testing.T) { + q := New() + defer q.Close() + + job := newJob("pong") + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + _, err := q.Enqueue(ctx, job) + if err == nil { + t.Fatal("expected Enqueue to return a context error once its deadline passes with nobody claiming the job") + } + + // The job must have been removed: a subsequent claim attempt with a + // short-lived context must time out rather than finding it. + claimCtx, claimCancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer claimCancel() + _, err = q.Claim(claimCtx, []string{"pong"}, "fin-1") + if err == nil { + t.Fatal("expected no job to be claimable after its Enqueue context expired") + } +} + +func TestEnqueueContextDoneRemovesLeasedJob(t *testing.T) { + q := New() + defer q.Close() + + job := newJob("pong") + enqueueCtx, enqueueCancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer enqueueCancel() + + done := make(chan struct{}) + go func() { + _, _ = q.Enqueue(enqueueCtx, job) + close(done) + }() + + claimCtx, claimCancel := context.WithTimeout(context.Background(), time.Second) + defer claimCancel() + _, err := q.Claim(claimCtx, []string{"pong"}, "fin-1") + if err != nil { + t.Fatal(err) + } + + <-done // wait for the Enqueue caller's deadline to pass + + // The Fin that claimed the job before it was abandoned should no longer + // be able to submit a result for it. + err = q.Submit(job.JobId, "fin-1", fin.JobResult{State: fin.JobStateSuccess}) + if err == nil { + t.Fatal("expected Submit to fail: the job was removed once its Enqueue context expired") + } +} + +func TestSubmitFailsForUnknownJob(t *testing.T) { + q := New() + defer q.Close() + + err := q.Submit(uuid.New(), "fin-1", fin.JobResult{State: fin.JobStateSuccess}) + if err == nil { + t.Fatal("expected Submit to fail for a job id that was never enqueued") + } +} + +func TestSubmitFailsWhenLeasedToADifferentFin(t *testing.T) { + q := New() + defer q.Close() + + job := newJob("pong") + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + go func() { _, _ = q.Enqueue(ctx, job) }() + + _, err := q.Claim(ctx, []string{"pong"}, "fin-1") + if err != nil { + t.Fatal(err) + } + + err = q.Submit(job.JobId, "fin-2", fin.JobResult{State: fin.JobStateSuccess}) + if err == nil { + t.Fatal("expected Submit to fail: job is leased to fin-1, not fin-2") + } +} + +func TestExpiredLeaseIsRequeuedForAnotherFin(t *testing.T) { + q := New() + defer q.Close() + + job := newJob("pong") + job.LeaseExpiresInSeconds = 0 // forces the minimum, still short, lease via a tiny override below + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + go func() { _, _ = q.Enqueue(ctx, job) }() + + claimed, err := q.Claim(ctx, []string{"pong"}, "fin-1") + if err != nil { + t.Fatal(err) + } + assert.Equal(t, claimed.JobId, job.JobId) + + // Force the lease to already be expired, instead of waiting out the + // real default lease duration, then let the sweeper's next tick pick it + // up. + q.mu.Lock() + if e, ok := q.leased[job.JobId]; ok { + e.leaseExpiry = time.Now().Add(-time.Second) + } + q.mu.Unlock() + + claimCtx, claimCancel := context.WithTimeout(context.Background(), 3*time.Second) + defer claimCancel() + reclaimed, err := q.Claim(claimCtx, []string{"pong"}, "fin-2") + if err != nil { + t.Fatal("expected the expired lease to be requeued and claimable by another fin:", err) + } + assert.Equal(t, reclaimed.JobId, job.JobId) + + err = q.Submit(job.JobId, "fin-2", fin.JobResult{State: fin.JobStateSuccess}) + if err != nil { + t.Fatal(err) + } +} + +func TestExtendLeasePreventsExpiryRequeue(t *testing.T) { + q := New() + defer q.Close() + + job := newJob("pong") + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + go func() { _, _ = q.Enqueue(ctx, job) }() + + _, err := q.Claim(ctx, []string{"pong"}, "fin-1") + if err != nil { + t.Fatal(err) + } + + // Simulate a lease about to expire, then extend it. + q.mu.Lock() + q.leased[job.JobId].leaseExpiry = time.Now().Add(10 * time.Millisecond) + q.mu.Unlock() + + if err := q.ExtendLease(job.JobId, "fin-1", 5); err != nil { + t.Fatal(err) + } + + // Give the sweeper a couple of ticks to run; the job must still be + // leased to fin-1, not requeued. + time.Sleep(2*defaultSweepInterval + 100*time.Millisecond) + + err = q.Submit(job.JobId, "fin-1", fin.JobResult{State: fin.JobStateSuccess}) + if err != nil { + t.Fatal("expected job to still be leased to fin-1 after ExtendLease:", err) + } +} + +func TestExtendLeaseFailsForUnknownJobOrWrongFin(t *testing.T) { + q := New() + defer q.Close() + + if err := q.ExtendLease(uuid.New(), "fin-1", 30); err == nil { + t.Fatal("expected ExtendLease to fail for an unknown job id") + } + + job := newJob("pong") + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + go func() { _, _ = q.Enqueue(ctx, job) }() + + _, err := q.Claim(ctx, []string{"pong"}, "fin-1") + if err != nil { + t.Fatal(err) + } + + if err := q.ExtendLease(job.JobId, "fin-2", 30); err == nil { + t.Fatal("expected ExtendLease to fail: job is leased to fin-1, not fin-2") + } +} diff --git a/internal/workflow/capability/fin/token/credentials.go b/internal/workflow/capability/fin/token/credentials.go new file mode 100644 index 00000000..ce1f336a --- /dev/null +++ b/internal/workflow/capability/fin/token/credentials.go @@ -0,0 +1,45 @@ +// Package token generates and hashes the bearer credentials used by the +// Fin protocol (fin_token, and the registration token comparison). Tokens +// are high-entropy random secrets; SOARCA never persists one in plaintext +// (see pkg/models/fin.Record.FinTokenHash) - only its SHA-256 hash, which +// is what callers compare/look up against on every authenticated call. +package token + +import ( + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/hex" + "errors" +) + +// tokenBytes is the amount of random entropy in a generated token, before +// hex-encoding (32 bytes = 256 bits). +const tokenBytes = 32 + +// Generate returns a new, high-entropy random token suitable for use as a +// fin_token. +func Generate() (string, error) { + buf := make([]byte, tokenBytes) + if _, err := rand.Read(buf); err != nil { + return "", errors.New("failed to generate random token: " + err.Error()) + } + return hex.EncodeToString(buf), nil +} + +// Hash returns the SHA-256 hash (hex-encoded) of token, for storage/ +// lookup/comparison instead of the plaintext value. No per-token salt is +// used: token itself is already a high-entropy random secret (see +// Generate), so unlike a user password hash, there is no risk of a +// precomputed dictionary/rainbow-table attack to defend against. +func Hash(token string) string { + sum := sha256.Sum256([]byte(token)) + return hex.EncodeToString(sum[:]) +} + +// Equal does a constant-time comparison of two tokens (e.g. a presented +// registration token against the configured one), to avoid leaking timing +// information about how much of a prefix matched. +func Equal(a string, b string) bool { + return subtle.ConstantTimeCompare([]byte(a), []byte(b)) == 1 +} diff --git a/internal/workflow/capability/fin/token/token_test.go b/internal/workflow/capability/fin/token/token_test.go new file mode 100644 index 00000000..d868efca --- /dev/null +++ b/internal/workflow/capability/fin/token/token_test.go @@ -0,0 +1,42 @@ +package token + +import ( + "testing" + + "github.com/go-playground/assert/v2" +) + +func TestGenerateProducesHighEntropyDistinctTokens(t *testing.T) { + first, err := Generate() + if err != nil { + t.Fatal(err) + } + second, err := Generate() + if err != nil { + t.Fatal(err) + } + if first == second { + t.Fatal("expected two generated tokens to differ") + } + assert.Equal(t, len(first), tokenBytes*2) // hex-encoded +} + +func TestHashIsDeterministicAndDistinct(t *testing.T) { + hashA := Hash("token-a") + hashAAgain := Hash("token-a") + hashB := Hash("token-b") + + assert.Equal(t, hashA, hashAAgain) + if hashA == hashB { + t.Fatal("expected different tokens to hash differently") + } + if hashA == "token-a" { + t.Fatal("expected Hash to not return the plaintext token") + } +} + +func TestEqual(t *testing.T) { + assert.Equal(t, Equal("secret", "secret"), true) + assert.Equal(t, Equal("secret", "different"), false) + assert.Equal(t, Equal("", ""), true) +} diff --git a/internal/workflow/capability/http/capability.go b/internal/workflow/capability/http/capability.go new file mode 100644 index 00000000..6dad646f --- /dev/null +++ b/internal/workflow/capability/http/capability.go @@ -0,0 +1,86 @@ +package http + +import ( + "reflect" + "soarca/internal/logger" + "soarca/internal/workflow/capability" + "soarca/pkg/cacao" + "soarca/internal/runs/model" + "soarca/pkg/utils/http" +) + +// Receive HTTP API command data from decomposer/executer +// Validate HTTP API call +// Run HTTP API call +// Return response + +const ( + httpApiResultVariableName = "__soarca_http_api_result__" + httpApiCapabilityName = "soarca-http-api" +) + +type HttpCapability struct { + soarca_http_request http.IHttpRequest +} + +type Empty struct{} + +var component = reflect.TypeOf(Empty{}).PkgPath() +var log *logger.Log + +func init() { + log = logger.Logger(component, logger.Info, "", logger.Json) +} + +func New(httpRequest http.IHttpRequest) *HttpCapability { + return &HttpCapability{soarca_http_request: httpRequest} +} + +func (httpCapability *HttpCapability) GetType() string { + return httpApiCapabilityName +} + +func (httpCapability *HttpCapability) Execute( + metadata run.Metadata, + context capability.Context) (cacao.Variables, error) { + + // This capability performs commands against a target; a step declaring + // zero targets has nothing to run against, so skip without error. + if len(context.Targets) == 0 { + return cacao.NewVariables(), nil + } + targets := context.Targets + + returnVariables := cacao.NewVariables() + var stepErr error + + for _, resolvedTarget := range targets { + target := resolvedTarget.Target + auth := resolvedTarget.Authentication + + for _, command := range context.Commands { + soarca_http_options := http.HttpOptions{ + Target: &target, + Command: &command, + Auth: &auth, + } + + responseBytes, err := httpCapability.soarca_http_request.Request(soarca_http_options) + if err != nil { + log.Error(err) + stepErr = err + // Abort this target's remaining commands on first failure, + // but keep processing the other targets. + break + } + respString := string(responseBytes) + variable := cacao.Variable{Type: cacao.VariableTypeString, + Name: httpApiResultVariableName, + Value: respString} + + returnVariables.Merge(cacao.NewVariables(variable)) + } + } + + return returnVariables, stepErr +} diff --git a/pkg/core/capability/http/http_test.go b/internal/workflow/capability/http/http_test.go similarity index 67% rename from pkg/core/capability/http/http_test.go rename to internal/workflow/capability/http/http_test.go index ce3b71e5..c1881094 100644 --- a/pkg/core/capability/http/http_test.go +++ b/internal/workflow/capability/http/http_test.go @@ -6,9 +6,9 @@ package http import ( "errors" - "soarca/pkg/core/capability" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" + "soarca/internal/workflow/capability" + "soarca/pkg/cacao" + "soarca/internal/runs/model" http_request "soarca/pkg/utils/http" mock_request "soarca/test/unittest/mocks/mock_utils/http" "testing" @@ -42,10 +42,10 @@ func TestHTTPOptionsCorrectlyGenerated(t *testing.T) { Value: "", } - var executionId, _ = uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + var runId, _ = uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") var playbookId, _ = uuid.Parse("d09351a2-a075-40c8-8054-0b7c423db83f") var stepId, _ = uuid.Parse("81eff59f-d084-4324-9e0a-59e353dbd28f") - metadata := execution.Metadata{ExecutionId: executionId, PlaybookId: playbookId.String(), StepId: stepId.String()} + metadata := run.Metadata{RunId: runId, PlaybookId: playbookId.String(), StepId: stepId.String()} httpOptions := http_request.HttpOptions{ Command: &command, @@ -58,10 +58,9 @@ func TestHTTPOptionsCorrectlyGenerated(t *testing.T) { mock_http_request.On("Request", httpOptions).Return(payload_byte, nil) data := capability.Context{ - Command: command, - Authentication: oauth2_info, - Target: target, - Variables: cacao.NewVariables(variable1), + Commands: []cacao.Command{command}, + Targets: []capability.ResolvedTarget{{Target: target, Authentication: oauth2_info}}, + Variables: cacao.NewVariables(variable1), } results, err := httpCapability.Execute( @@ -96,10 +95,10 @@ func TestHTTPOptionsEmptyAuth(t *testing.T) { Value: "", } - var executionId, _ = uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + var runId, _ = uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") var playbookId, _ = uuid.Parse("d09351a2-a075-40c8-8054-0b7c423db83f") var stepId, _ = uuid.Parse("81eff59f-d084-4324-9e0a-59e353dbd28f") - metadata := execution.Metadata{ExecutionId: executionId, PlaybookId: playbookId.String(), StepId: stepId.String()} + metadata := run.Metadata{RunId: runId, PlaybookId: playbookId.String(), StepId: stepId.String()} empty_auth := new(cacao.AuthenticationInformation) httpOptions := http_request.HttpOptions{ @@ -113,10 +112,9 @@ func TestHTTPOptionsEmptyAuth(t *testing.T) { mock_http_request.On("Request", httpOptions).Return(payload_byte, nil) data := capability.Context{ - Command: command, - Authentication: *empty_auth, - Target: target, - Variables: cacao.NewVariables(variable1), + Commands: []cacao.Command{command}, + Targets: []capability.ResolvedTarget{{Target: target, Authentication: *empty_auth}}, + Variables: cacao.NewVariables(variable1), } results, err := httpCapability.Execute( @@ -153,10 +151,10 @@ func TestHTTPOptionsEmptyCommand(t *testing.T) { Value: "", } - var executionId, _ = uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + var runId, _ = uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") var playbookId, _ = uuid.Parse("d09351a2-a075-40c8-8054-0b7c423db83f") var stepId, _ = uuid.Parse("81eff59f-d084-4324-9e0a-59e353dbd28f") - metadata := execution.Metadata{ExecutionId: executionId, PlaybookId: playbookId.String(), StepId: stepId.String()} + metadata := run.Metadata{RunId: runId, PlaybookId: playbookId.String(), StepId: stepId.String()} httpOptions := http_request.HttpOptions{ Command: empty_command, @@ -168,10 +166,9 @@ func TestHTTPOptionsEmptyCommand(t *testing.T) { mock_http_request.On("Request", httpOptions).Return([]byte{}, expected_error) data := capability.Context{ - Command: *empty_command, - Authentication: oauth2_info, - Target: target, - Variables: cacao.NewVariables(variable1), + Commands: []cacao.Command{*empty_command}, + Targets: []capability.ResolvedTarget{{Target: target, Authentication: oauth2_info}}, + Variables: cacao.NewVariables(variable1), } results, err := httpCapability.Execute( @@ -186,3 +183,32 @@ func TestHTTPOptionsEmptyCommand(t *testing.T) { mock_http_request.AssertExpectations(t) } + +func TestHTTPExecuteNoTargetsSkipsWithoutError(t *testing.T) { + mock_http_request := new(mock_request.MockHttpRequest) + httpCapability := New(mock_http_request) + + command := cacao.Command{ + Type: "http-api", + Command: "POST / HTTP/1.1", + } + + var runId, _ = uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + var playbookId, _ = uuid.Parse("d09351a2-a075-40c8-8054-0b7c423db83f") + var stepId, _ = uuid.Parse("81eff59f-d084-4324-9e0a-59e353dbd28f") + metadata := run.Metadata{RunId: runId, PlaybookId: playbookId.String(), StepId: stepId.String()} + + data := capability.Context{ + Commands: []cacao.Command{command}, + Targets: []capability.ResolvedTarget{}, + } + + results, err := httpCapability.Execute(metadata, data) + if err != nil { + t.Log(err) + t.Fail() + } + assert.Equal(t, len(results), 0) + + mock_http_request.AssertNotCalled(t, "Request") +} diff --git a/pkg/core/capability/manual/manual.go b/internal/workflow/capability/manual/capability.go similarity index 63% rename from pkg/core/capability/manual/manual.go rename to internal/workflow/capability/manual/capability.go index b9803969..4d6b0887 100644 --- a/pkg/core/capability/manual/manual.go +++ b/internal/workflow/capability/manual/capability.go @@ -5,11 +5,12 @@ import ( "errors" "reflect" "soarca/internal/logger" - "soarca/pkg/core/capability" - "soarca/pkg/core/capability/manual/interaction" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" - manualModel "soarca/pkg/models/manual" + "soarca/internal/workflow/capability" + "soarca/internal/workflow/capability/manual/inbox" + "soarca/pkg/cacao" + manualModel "soarca/internal/manual/model" + "soarca/internal/runs/model" + "soarca/pkg/utils" "time" ) @@ -21,11 +22,10 @@ var ( const ( manualResultVariableName = "__soarca_manual_result__" manualCapabilityName = "soarca-manual" - fallbackTimeout = time.Minute * 1 ) -func New(controller interaction.ICapabilityInteraction) ManualCapability { - return ManualCapability{interaction: controller} +func New(controller inbox.Dispatcher) ManualCapability { + return ManualCapability{inbox: controller} } func init() { @@ -33,7 +33,7 @@ func init() { } type ManualCapability struct { - interaction interaction.ICapabilityInteraction + inbox inbox.Dispatcher } func (manual *ManualCapability) GetType() string { @@ -41,7 +41,7 @@ func (manual *ManualCapability) GetType() string { } func (manual *ManualCapability) Execute( - metadata execution.Metadata, + metadata run.Metadata, commandContext capability.Context) (cacao.Variables, error) { command := manualModel.CommandInfo{ @@ -57,10 +57,10 @@ func (manual *ManualCapability) Execute( // One channel per Execute() invocation. Async manual capability Execute() invocations can thus // use separate channels per each specific manual command, preventing manual returned args interfering - channel := make(chan manualModel.InteractionResponse) + channel := make(chan manualModel.Response) defer close(channel) - err := manual.interaction.Queue(command, manualModel.ManualCapabilityCommunication{ + err := manual.inbox.Queue(command, manualModel.Waiter{ Channel: channel, TimeoutContext: ctx, }) @@ -70,6 +70,17 @@ func (manual *ManualCapability) Execute( } result, err := manual.awaitUserInput(channel, ctx) + + // Deregister synchronously, before returning, so a subsequent + // re-run of this step (e.g. the next iteration of a while-loop + // body) can never race the async cleanup goroutine + // Inbox.Queue also starts as a backstop. Keyed on this + // invocation's StepRunId, so it never touches a different + // invocation's still-pending entry, even one sharing the same StepId. + if deregErr := manual.inbox.Deregister(metadata); deregErr != nil { + log.Trace("manual command already deregistered: ", deregErr) + } + if err != nil { return cacao.NewVariables(), err } @@ -77,7 +88,7 @@ func (manual *ManualCapability) Execute( } -func (manual *ManualCapability) awaitUserInput(channel chan manualModel.InteractionResponse, ctx context.Context) (cacao.Variables, error) { +func (manual *ManualCapability) awaitUserInput(channel chan manualModel.Response, ctx context.Context) (cacao.Variables, error) { for { select { @@ -96,8 +107,9 @@ func (manual *ManualCapability) awaitUserInput(channel chan manualModel.Interact func (manual *ManualCapability) getTimeoutValue(userTimeout int) time.Duration { if userTimeout == 0 { - log.Warning("timeout is not set or set to 0 fallback timeout of 1 minute is used to complete step") - return fallbackTimeout + fallback := utils.DefaultStepTimeout() + log.Warning("timeout is not set or set to 0, fallback timeout of ", fallback, " is used to complete step") + return fallback } return time.Duration(userTimeout) * time.Millisecond } diff --git a/internal/workflow/capability/manual/inbox/inbox.go b/internal/workflow/capability/manual/inbox/inbox.go new file mode 100644 index 00000000..a6c3f807 --- /dev/null +++ b/internal/workflow/capability/manual/inbox/inbox.go @@ -0,0 +1,286 @@ +package inbox + +import ( + "context" + "errors" + "fmt" + "reflect" + "soarca/internal/logger" + "soarca/internal/registry" + "soarca/pkg/cacao" + "soarca/internal/manual/model" + "soarca/internal/runs/model" +) + +type Empty struct{} + +var component = reflect.TypeOf(Empty{}).PkgPath() +var log *logger.Log + +func init() { + log = logger.Logger(component, logger.Info, "", logger.Json) +} + +type Notifier interface { + Notify(command manual.Notification, channel chan manual.Response) +} + +type Dispatcher interface { + Queue(command manual.CommandInfo, manualComms manual.Waiter) error + // Deregister removes the pending interaction for metadata, if still + // present. Callers that own the full lifecycle of a queued command + // (i.e. they know when it has been resolved or has timed out) should + // call this synchronously as soon as that happens, so a subsequent + // re-run of the same step - e.g. a step inside a while-loop body - + // never races an async cleanup routine trying to remove the same, + // already-superseded entry. + Deregister(metadata run.Metadata) error +} + +type Store interface { + GetPendingCommands() ([]manual.CommandInfo, error) + // GetPendingCommand looks up one specific pending command by its + // StepRunId. Multiple pending commands may legitimately share the + // same StepId - e.g. overlapping while-loop iterations, or (once + // implemented) concurrent parallel branches converging on the same + // step - so StepId alone cannot identify a single pending command; it's + // up to the caller (UI/integrator) to disambiguate between several + // pending commands for the same StepId using StepRunId. + GetPendingCommand(metadata run.Metadata) (manual.CommandInfo, error) + PostContinue(response manual.Response) error +} + +type Inbox struct { + pending *registry.Registry[manual.PendingCommand] // Keyed on [runID][stepRunID] + Notifiers []Notifier +} + +func New(manualIntegrations []Notifier) *Inbox { + return &Inbox{ + pending: registry.New[manual.PendingCommand](), + Notifiers: manualIntegrations, + } +} + +// ############################################################################ +// Dispatcher implementation +// ############################################################################ +func (manualController *Inbox) Queue(command manual.CommandInfo, manualComms manual.Waiter) error { + + err := manualController.registerPendingCommand(command, manualComms.Channel) + if err != nil { + return err + } + + if _, ok := manualComms.TimeoutContext.Deadline(); !ok { + return errors.New("manual command does not have a deadline") + } + + // Copy and type conversion + integrationCommand := manual.Notification(command) + + // One response channel for all integrations + integrationChannel := make(chan manual.Response) + + for _, notifier := range manualController.Notifiers { + go notifier.Notify(integrationCommand, integrationChannel) + } + + // Backstop cleanup: removes the pending interaction if the caller + // driving this command (e.g. ManualCapability.Execute) never calls + // Deregister itself - for instance if Queue is used directly, as in + // this package's own tests. If Deregister has already been called + // synchronously by the caller, this is a harmless no-op (see + // handleManualCommandResponse). + go manualController.handleManualCommandResponse(command, manualComms) + + return nil +} + +// Deregister removes the pending interaction for metadata, if still +// present. It is safe to call even if the interaction was already +// removed (e.g. by the internal timeout/completion cleanup goroutine +// racing this call): in that case it returns +// manual.ErrorPendingCommandNotFound, which callers can treat as a +// benign, expected outcome rather than a failure. +func (manualController *Inbox) Deregister(metadata run.Metadata) error { + return manualController.removeCommandFromPending(metadata) +} + +func (manualController *Inbox) handleManualCommandResponse(command manual.CommandInfo, manualComms manual.Waiter) { + log.Trace( + fmt.Sprintf( + "goroutine handling command response %s, %s (step run %s) has started", + command.Metadata.RunId.String(), command.Metadata.StepId, command.Metadata.StepRunId.String())) + defer log.Trace( + fmt.Sprintf( + "goroutine handling command response %s, %s (step run %s) has ended", + command.Metadata.RunId.String(), command.Metadata.StepId, command.Metadata.StepRunId.String())) + + // Wait for either timeout or response + <-manualComms.TimeoutContext.Done() + if manualComms.TimeoutContext.Err() == context.DeadlineExceeded { + log.Info("manual command timed out. deregistering associated pending command") + } else if manualComms.TimeoutContext.Err() == context.Canceled { + log.Info("manual command completed. deregistering associated pending command") + } + err := manualController.removeCommandFromPending(command.Metadata) + if err != nil { + log.Warning(err) + log.Warning("manual command not found among pending ones. should be already resolved") + return + } +} + +// ############################################################################ +// Store implementation +// ############################################################################ +func (manualController *Inbox) GetPendingCommands() ([]manual.CommandInfo, error) { + log.Trace("getting pending manual commands") + return manualController.getAllPendingCommandsInfo(), nil +} + +func (manualController *Inbox) GetPendingCommand(metadata run.Metadata) (manual.CommandInfo, error) { + log.Trace("getting pending manual command") + pending, err := manualController.getPendingCommand(metadata) + return pending.CommandInfo, err +} + +func (manualController *Inbox) PostContinue(response manual.Response) error { + log.Trace("completing manual command") + + // If not in there, it means it was already solved, or expired + pendingEntry, err := manualController.getPendingCommand(response.Metadata) + if err != nil { + log.Warning(err) + return err + } + + warnings, err := manualController.validateMatchingOutArgs(pendingEntry, response.OutArgsVariables) + if err != nil { + return err + } + + //Then put outArgs back into manualCapabilityChannel + // Copy result and conversion back to response format + log.Trace("pushing assigned variables in manual capability channel") + pendingEntry.Channel <- response + + if len(warnings) > 0 { + for _, warning := range warnings { + log.Warning(warning) + } + } + + return nil +} + +// ############################################################################ +// Utilities and functionalities +// ############################################################################ +func (manualController *Inbox) registerPendingCommand(command manual.CommandInfo, manualChan chan manual.Response) error { + + commandInfo := manual.CommandInfo{ + Metadata: command.Metadata, + Context: command.Context, + OutArgsVariables: command.OutArgsVariables, + } + + entry := manual.PendingCommand{ + CommandInfo: commandInfo, + Channel: manualChan, + } + + err := manualController.pending.Register( + commandInfo.Metadata.RunId.String(), + commandInfo.Metadata.StepRunId.String(), + entry, + ) + if err != nil { + var alreadyRegistered registry.ErrAlreadyRegistered + if errors.As(err, &alreadyRegistered) { + // Practically unreachable in normal operation: StepRunId + // is a fresh UUID minted once per step invocation + // (decomposer.newStepMetadata), so a collision here means Queue + // was called twice for the exact same invocation. + err := fmt.Errorf( + "a manual command is already pending for run %s, step run %s (step %s)", + commandInfo.Metadata.RunId.String(), commandInfo.Metadata.StepRunId.String(), commandInfo.Metadata.StepId) + log.Error(err) + return err + } + return err + } + + return nil +} + +func (manualController *Inbox) getAllPendingCommandsInfo() []manual.CommandInfo { + entries := manualController.pending.List() + allPendingCommands := make([]manual.CommandInfo, 0, len(entries)) + for _, entry := range entries { + allPendingCommands = append(allPendingCommands, entry.CommandInfo) + } + return allPendingCommands +} + +func (manualController *Inbox) getPendingCommand(commandMetadata run.Metadata) (manual.PendingCommand, error) { + entry, err := manualController.pending.Get(commandMetadata.RunId.String(), commandMetadata.StepRunId.String()) + if err != nil { + var outerNotFound registry.ErrOuterKeyNotFound + if errors.As(err, &outerNotFound) { + errMsg := fmt.Sprintf("no pending commands found for run %s", commandMetadata.RunId.String()) + return manual.PendingCommand{}, manual.ErrorPendingCommandNotFound{Err: errMsg} + } + var innerNotFound registry.ErrInnerKeyNotFound + if errors.As(err, &innerNotFound) { + errMsg := fmt.Sprintf("no pending command found for run %s -> step run %s", + commandMetadata.RunId.String(), + commandMetadata.StepRunId.String(), + ) + return manual.PendingCommand{}, manual.ErrorPendingCommandNotFound{Err: errMsg} + } + return manual.PendingCommand{}, err + } + return entry, nil +} + +func (manualController *Inbox) removeCommandFromPending(commandMetadata run.Metadata) error { + _, err := manualController.getPendingCommand(commandMetadata) + if err != nil { + return err + } + // Errors from Remove are already covered by the getPendingCommand + // check above, so this pair (run id, step run id) is known + // to exist. + return manualController.pending.Remove(commandMetadata.RunId.String(), commandMetadata.StepRunId.String()) +} + +func (manualController *Inbox) validateMatchingOutArgs(pendingEntry manual.PendingCommand, responseOutArgs cacao.Variables) ([]string, error) { + warns := []string{} + var err error = nil + for varName, variable := range responseOutArgs { + // first check that out args provided match the variables + if _, ok := pendingEntry.CommandInfo.OutArgsVariables[varName]; !ok { + err = fmt.Errorf("provided out arg %s does not match any intended out arg", varName) + return warns, manual.ErrorNonMatchingOutArgs{Err: err.Error()} + + } + // then warn if any value outside "value" has changed + if pending, ok := pendingEntry.CommandInfo.OutArgsVariables[varName]; ok { + if variable.Constant != pending.Constant { + warns = append(warns, fmt.Sprintf("provided out arg %s has different value for 'Constant' property of intended out arg. This different value is ignored.", varName)) + } + if variable.Description != pending.Description { + warns = append(warns, fmt.Sprintf("provided out arg %s has different value for 'Description' property of intended out arg. This different value is ignored.", varName)) + } + if variable.External != pending.External { + warns = append(warns, fmt.Sprintf("provided out arg %s has different value for 'External' property of intended out arg. This different value is ignored.", varName)) + } + if variable.Type != pending.Type { + warns = append(warns, fmt.Sprintf("provided out arg %s has different value for 'Type' property of intended out arg. This different value is ignored.", varName)) + } + } + } + return warns, err +} diff --git a/pkg/core/capability/manual/interaction/interaction_test.go b/internal/workflow/capability/manual/inbox/inbox_test.go similarity index 61% rename from pkg/core/capability/manual/interaction/interaction_test.go rename to internal/workflow/capability/manual/inbox/inbox_test.go index d7916ac2..8781eaa8 100644 --- a/pkg/core/capability/manual/interaction/interaction_test.go +++ b/internal/workflow/capability/manual/inbox/inbox_test.go @@ -1,4 +1,4 @@ -package interaction +package inbox import ( "context" @@ -6,10 +6,10 @@ import ( "errors" "fmt" "reflect" - "soarca/pkg/core/capability" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" - manualModel "soarca/pkg/models/manual" + "soarca/internal/workflow/capability" + "soarca/pkg/cacao" + manualModel "soarca/internal/manual/model" + "soarca/internal/runs/model" "sort" "strings" "testing" @@ -21,12 +21,12 @@ import ( ) func TestQueue(t *testing.T) { - interaction := New([]IInteractionIntegrationNotifier{}) + interaction := New([]Notifier{}) testCtx, testCancel := context.WithTimeout(context.Background(), 10*time.Millisecond) defer testCancel() - testCapComms := manualModel.ManualCapabilityCommunication{ - Channel: make(chan manualModel.InteractionResponse), + testCapComms := manualModel.Waiter{ + Channel: make(chan manualModel.Response), TimeoutContext: testCtx, } @@ -39,12 +39,12 @@ func TestQueue(t *testing.T) { } func TestQueueFailWithoutTimeout(t *testing.T) { - interaction := New([]IInteractionIntegrationNotifier{}) + interaction := New([]Notifier{}) testCommand := manualModel.CommandInfo{} - testCapComms := manualModel.ManualCapabilityCommunication{ - Channel: make(chan manualModel.InteractionResponse), + testCapComms := manualModel.Waiter{ + Channel: make(chan manualModel.Response), TimeoutContext: context.WithoutCancel(context.Background()), } err := interaction.Queue(testCommand, testCapComms) @@ -52,7 +52,7 @@ func TestQueueFailWithoutTimeout(t *testing.T) { } func TestQueueExitOnTimeout(t *testing.T) { - interaction := New([]IInteractionIntegrationNotifier{}) + interaction := New([]Notifier{}) timeout := 30 * time.Millisecond testCtx, testCancel := context.WithTimeout(context.Background(), timeout) defer testCancel() @@ -60,8 +60,8 @@ func TestQueueExitOnTimeout(t *testing.T) { hook := NewTestLogHook() log.Logger.AddHook(hook) - testCapComms := manualModel.ManualCapabilityCommunication{ - Channel: make(chan manualModel.InteractionResponse), + testCapComms := manualModel.Waiter{ + Channel: make(chan manualModel.Response), TimeoutContext: testCtx, } @@ -86,17 +86,17 @@ func TestQueueExitOnTimeout(t *testing.T) { } -func TestRegisterRetrieveNewPendingInteraction(t *testing.T) { - interaction := New([]IInteractionIntegrationNotifier{}) - testChan := make(chan manualModel.InteractionResponse) +func TestRegisterRetrieveNewPendingCommand(t *testing.T) { + interaction := New([]Notifier{}) + testChan := make(chan manualModel.Response) defer close(testChan) - err := interaction.registerPendingInteraction(testInteractionCommand, testChan) + err := interaction.registerPendingCommand(testInteractionCommand, testChan) if err != nil { t.Log(err) t.Fail() } - retrievedCommand, err := interaction.getPendingInteraction(testMetadata) + retrievedCommand, err := interaction.getPendingCommand(testMetadata) if err != nil { t.Log(err) t.Fail() @@ -108,7 +108,7 @@ func TestRegisterRetrieveNewPendingInteraction(t *testing.T) { testChan, ) - // Execution metadata + // Run metadata assert.Equal(t, retrievedCommand.CommandInfo.Metadata, testInteractionCommand.Metadata, @@ -125,13 +125,13 @@ func TestRegisterRetrieveNewPendingInteraction(t *testing.T) { ) } -func TestGetAllPendingInteractions(t *testing.T) { - interaction := New([]IInteractionIntegrationNotifier{}) - testChan := make(chan manualModel.InteractionResponse) +func TestGetAllPendingCommands(t *testing.T) { + interaction := New([]Notifier{}) + testChan := make(chan manualModel.Response) defer close(testChan) localTestInteractionCommand := testInteractionCommand - err := interaction.registerPendingInteraction(localTestInteractionCommand, testChan) + err := interaction.registerPendingCommand(localTestInteractionCommand, testChan) if err != nil { t.Log(err) t.Fail() @@ -139,9 +139,9 @@ func TestGetAllPendingInteractions(t *testing.T) { testNewInteractionCommand := localTestInteractionCommand newExecId := "50b6d52c-6efc-4516-a242-dfbc5c89d421" - testNewInteractionCommand.Metadata.ExecutionId = uuid.MustParse(newExecId) + testNewInteractionCommand.Metadata.RunId = uuid.MustParse(newExecId) - err = interaction.registerPendingInteraction(testNewInteractionCommand, testChan) + err = interaction.registerPendingCommand(testNewInteractionCommand, testChan) if err != nil { t.Log(err) t.Fail() @@ -150,12 +150,12 @@ func TestGetAllPendingInteractions(t *testing.T) { expectedInteractions := []manualModel.CommandInfo{localTestInteractionCommand, testNewInteractionCommand} receivedInteractions := interaction.getAllPendingCommandsInfo() - // Sort both slices by ExecutionId + // Sort both slices by RunId sort.Slice(expectedInteractions, func(i, j int) bool { - return expectedInteractions[i].Metadata.ExecutionId.String() < expectedInteractions[j].Metadata.ExecutionId.String() + return expectedInteractions[i].Metadata.RunId.String() < expectedInteractions[j].Metadata.RunId.String() }) sort.Slice(receivedInteractions, func(i, j int) bool { - return receivedInteractions[i].Metadata.ExecutionId.String() < receivedInteractions[j].Metadata.ExecutionId.String() + return receivedInteractions[i].Metadata.RunId.String() < receivedInteractions[j].Metadata.RunId.String() }) receivedInteractionsJson, err := json.MarshalIndent(receivedInteractions, "", " ") @@ -186,14 +186,14 @@ func TestGetAllPendingInteractions(t *testing.T) { } } -func TestRegisterRetrieveSameExecutionMultiplePendingInteraction(t *testing.T) { - interaction := New([]IInteractionIntegrationNotifier{}) - testChan := make(chan manualModel.InteractionResponse) +func TestRegisterRetrieveSameRunMultiplePendingCommand(t *testing.T) { + interaction := New([]Notifier{}) + testChan := make(chan manualModel.Response) defer close(testChan) localTestInteractionCommand := testInteractionCommand - err := interaction.registerPendingInteraction(localTestInteractionCommand, testChan) + err := interaction.registerPendingCommand(localTestInteractionCommand, testChan) if err != nil { t.Log(err) t.Fail() @@ -202,31 +202,85 @@ func TestRegisterRetrieveSameExecutionMultiplePendingInteraction(t *testing.T) { testNewInteractionCommandSecond := localTestInteractionCommand newStepId2 := "test_second_step_id" testNewInteractionCommandSecond.Metadata.StepId = newStepId2 + testNewInteractionCommandSecond.Metadata.StepRunId = uuid.MustParse("22a6c41e-6efc-4516-a242-dfbc5c89d562") testNewInteractionCommandThird := localTestInteractionCommand newStepId3 := "test_third_step_id" testNewInteractionCommandThird.Metadata.StepId = newStepId3 + testNewInteractionCommandThird.Metadata.StepRunId = uuid.MustParse("33a6c41e-6efc-4516-a242-dfbc5c89d562") - err = interaction.registerPendingInteraction(testNewInteractionCommandSecond, testChan) + err = interaction.registerPendingCommand(testNewInteractionCommandSecond, testChan) if err != nil { t.Log(err) t.Fail() } - err = interaction.registerPendingInteraction(testNewInteractionCommandThird, testChan) + err = interaction.registerPendingCommand(testNewInteractionCommandThird, testChan) if err != nil { t.Log(err) t.Fail() } } +// Two invocations of the *same* StepId (e.g. two overlapping while-loop +// iterations, or - once implemented - two parallel branches converging on +// the same step) must be able to have their own independently pending +// manual command, each tracked and resolved by its own StepRunId. +func TestRegisterRetrieveSameStepIdDifferentStepRunIdPendingCommands(t *testing.T) { + interaction := New([]Notifier{}) + testChan := make(chan manualModel.Response) + defer close(testChan) + + firstInvocation := testInteractionCommand + secondInvocation := testInteractionCommand + secondInvocation.Metadata.StepRunId = uuid.MustParse("44a6c41e-6efc-4516-a242-dfbc5c89d562") + + // Same RunId and StepId, different StepRunId. + err := interaction.registerPendingCommand(firstInvocation, testChan) + if err != nil { + t.Log(err) + t.Fail() + } + err = interaction.registerPendingCommand(secondInvocation, testChan) + if err != nil { + t.Log(err) + t.Fail() + } + + firstRetrieved, err := interaction.getPendingCommand(firstInvocation.Metadata) + if err != nil { + t.Log(err) + t.Fail() + } + secondRetrieved, err := interaction.getPendingCommand(secondInvocation.Metadata) + if err != nil { + t.Log(err) + t.Fail() + } + + assert.Equal(t, firstRetrieved.CommandInfo.Metadata, firstInvocation.Metadata) + assert.Equal(t, secondRetrieved.CommandInfo.Metadata, secondInvocation.Metadata) + + // Resolving/removing one must not affect the other. + err = interaction.removeCommandFromPending(firstInvocation.Metadata) + if err != nil { + t.Log(err) + t.Fail() + } + _, err = interaction.getPendingCommand(secondInvocation.Metadata) + if err != nil { + t.Log("second invocation should still be pending after removing the first") + t.Fail() + } +} + func TestCopyOutArgsToVars(t *testing.T) { - interaction := New([]IInteractionIntegrationNotifier{}) + interaction := New([]Notifier{}) testCtx, testCancel := context.WithTimeout(context.Background(), 10*time.Millisecond) defer testCancel() - testCapComms := manualModel.ManualCapabilityCommunication{ - Channel: make(chan manualModel.InteractionResponse), + testCapComms := manualModel.Waiter{ + Channel: make(chan manualModel.Response), TimeoutContext: testCtx, } @@ -262,7 +316,7 @@ func TestCopyOutArgsToVars(t *testing.T) { func TestValidateMatchingOutArgs(t *testing.T) { - interaction := New([]IInteractionIntegrationNotifier{}) + interaction := New([]Notifier{}) respOutArg := cacao.Variable{ Type: "pears", @@ -285,13 +339,13 @@ func TestValidateMatchingOutArgs(t *testing.T) { } storedOutArgs := cacao.Variables{"__var2__": storedOutArg} - interactionStorageEntry := manualModel.InteractionStorageEntry{ + interactionStorageEntry := manualModel.PendingCommand{ CommandInfo: manualModel.CommandInfo{ Metadata: testMetadata, Context: capability.Context{}, OutArgsVariables: storedOutArgs, }, - Channel: make(chan manualModel.InteractionResponse), + Channel: make(chan manualModel.Response), } expectedLogEntry1 := "provided out arg __var2__ has different value for 'Constant' property of intended out arg. This different value is ignored." @@ -317,7 +371,7 @@ func TestValidateMatchingOutArgs(t *testing.T) { } func TestPostContinueFailOnNonexistingVariable(t *testing.T) { - interaction := New([]IInteractionIntegrationNotifier{}) + interaction := New([]Notifier{}) timeout := 500 * time.Millisecond testCtx, testCancel := context.WithTimeout(context.Background(), timeout) @@ -326,8 +380,8 @@ func TestPostContinueFailOnNonexistingVariable(t *testing.T) { hook := NewTestLogHook() log.Logger.AddHook(hook) - testCapComms := manualModel.ManualCapabilityCommunication{ - Channel: make(chan manualModel.InteractionResponse), + testCapComms := manualModel.Waiter{ + Channel: make(chan manualModel.Response), TimeoutContext: testCtx, } defer close(testCapComms.Channel) @@ -344,7 +398,7 @@ func TestPostContinueFailOnNonexistingVariable(t *testing.T) { Value: "now the value is bananas", } - outArgsUpdate := manualModel.InteractionResponse{ + outArgsUpdate := manualModel.Response{ Metadata: testMetadata, ResponseStatus: "success", ResponseError: nil, @@ -359,12 +413,12 @@ func TestPostContinueFailOnNonexistingVariable(t *testing.T) { assert.Equal(t, err, expectedErr) } -func TestRegisterRetrieveNewExecutionNewPendingInteraction(t *testing.T) { - interaction := New([]IInteractionIntegrationNotifier{}) - testChan := make(chan manualModel.InteractionResponse) +func TestRegisterRetrieveNewRunNewPendingCommand(t *testing.T) { + interaction := New([]Notifier{}) + testChan := make(chan manualModel.Response) defer close(testChan) - err := interaction.registerPendingInteraction(testInteractionCommand, testChan) + err := interaction.registerPendingCommand(testInteractionCommand, testChan) if err != nil { t.Log(err) t.Fail() @@ -372,9 +426,9 @@ func TestRegisterRetrieveNewExecutionNewPendingInteraction(t *testing.T) { testNewInteractionCommand := testInteractionCommand newExecId := "50b6d52c-6efc-4516-a242-dfbc5c89d421" - testNewInteractionCommand.Metadata.ExecutionId = uuid.MustParse(newExecId) + testNewInteractionCommand.Metadata.RunId = uuid.MustParse(newExecId) - err = interaction.registerPendingInteraction(testNewInteractionCommand, testChan) + err = interaction.registerPendingCommand(testNewInteractionCommand, testChan) if err != nil { t.Log(err) t.Fail() @@ -382,8 +436,8 @@ func TestRegisterRetrieveNewExecutionNewPendingInteraction(t *testing.T) { } func TestGetEmptyPendingCommand(t *testing.T) { - interaction := New([]IInteractionIntegrationNotifier{}) - testChan := make(chan manualModel.InteractionResponse) + interaction := New([]Notifier{}) + testChan := make(chan manualModel.Response) defer close(testChan) emptyCommandInfo, err := interaction.GetPendingCommand(testMetadata) @@ -393,7 +447,7 @@ func TestGetEmptyPendingCommand(t *testing.T) { } expectedErr := manualModel.ErrorPendingCommandNotFound{ - Err: "no pending commands found for execution " + + Err: "no pending commands found for run " + "61a6c41e-6efc-4516-a242-dfbc5c89d562", } @@ -401,120 +455,120 @@ func TestGetEmptyPendingCommand(t *testing.T) { assert.Equal(t, err, expectedErr) } -func TestFailOnRegisterSamePendingInteraction(t *testing.T) { - interaction := New([]IInteractionIntegrationNotifier{}) - testChan := make(chan manualModel.InteractionResponse) +func TestFailOnRegisterSamePendingCommand(t *testing.T) { + interaction := New([]Notifier{}) + testChan := make(chan manualModel.Response) defer close(testChan) - err := interaction.registerPendingInteraction(testInteractionCommand, testChan) + err := interaction.registerPendingCommand(testInteractionCommand, testChan) if err != nil { t.Log(err) t.Fail() } - err = interaction.registerPendingInteraction(testInteractionCommand, testChan) + err = interaction.registerPendingCommand(testInteractionCommand, testChan) if err == nil { t.Log(err) t.Fail() } - expectedErr := errors.New( - "a manual step is already pending for execution " + - "61a6c41e-6efc-4516-a242-dfbc5c89d562, step test_step_id. " + - "There can only be one pending manual command per action step", + expectedErr := fmt.Errorf( + "a manual command is already pending for run " + + "61a6c41e-6efc-4516-a242-dfbc5c89d562, step run " + + "11a6c41e-6efc-4516-a242-dfbc5c89d562 (step test_step_id)", ) assert.Equal(t, err, expectedErr) } -func TestFailOnRetrieveUnexistingExecutionInteraction(t *testing.T) { - interaction := New([]IInteractionIntegrationNotifier{}) - testChan := make(chan manualModel.InteractionResponse) +func TestFailOnRetrieveUnexistingRunInteraction(t *testing.T) { + interaction := New([]Notifier{}) + testChan := make(chan manualModel.Response) defer close(testChan) testDifferentMetadata := testMetadata newExecId := "50b6d52c-6efc-4516-a242-dfbc5c89d421" - testDifferentMetadata.ExecutionId = uuid.MustParse(newExecId) + testDifferentMetadata.RunId = uuid.MustParse(newExecId) - _, err := interaction.getPendingInteraction(testDifferentMetadata) + _, err := interaction.getPendingCommand(testDifferentMetadata) if err == nil { t.Log(err) t.Fail() } expectedErr := manualModel.ErrorPendingCommandNotFound{ - Err: "no pending commands found for execution 50b6d52c-6efc-4516-a242-dfbc5c89d421", + Err: "no pending commands found for run 50b6d52c-6efc-4516-a242-dfbc5c89d421", } assert.Equal(t, err, expectedErr) } func TestFailOnRetrieveNonExistingCommandInteraction(t *testing.T) { - interaction := New([]IInteractionIntegrationNotifier{}) - testChan := make(chan manualModel.InteractionResponse) + interaction := New([]Notifier{}) + testChan := make(chan manualModel.Response) defer close(testChan) - err := interaction.registerPendingInteraction(testInteractionCommand, testChan) + err := interaction.registerPendingCommand(testInteractionCommand, testChan) if err != nil { t.Log(err) t.Fail() } testDifferentMetadata := testMetadata - newStepId := "50b6d52c-6efc-4516-a242-dfbc5c89d421" - testDifferentMetadata.StepId = newStepId + newStepRunId := "50b6d52c-6efc-4516-a242-dfbc5c89d421" + testDifferentMetadata.StepRunId = uuid.MustParse(newStepRunId) - _, err = interaction.getPendingInteraction(testDifferentMetadata) + _, err = interaction.getPendingCommand(testDifferentMetadata) if err == nil { t.Log(err) t.Fail() } expectedErr := manualModel.ErrorPendingCommandNotFound{ - Err: "no pending commands found for execution " + + Err: "no pending command found for run " + "61a6c41e-6efc-4516-a242-dfbc5c89d562 -> " + - "step 50b6d52c-6efc-4516-a242-dfbc5c89d421", + "step run 50b6d52c-6efc-4516-a242-dfbc5c89d421", } assert.Equal(t, err, expectedErr) } func TestRemovePendingInteraciton(t *testing.T) { - interaction := New([]IInteractionIntegrationNotifier{}) - testChan := make(chan manualModel.InteractionResponse) + interaction := New([]Notifier{}) + testChan := make(chan manualModel.Response) defer close(testChan) - err := interaction.registerPendingInteraction(testInteractionCommand, testChan) + err := interaction.registerPendingCommand(testInteractionCommand, testChan) if err != nil { t.Log(err) t.Fail() } - pendingCommand, err := interaction.getPendingInteraction(testMetadata) + pendingCommand, err := interaction.getPendingCommand(testMetadata) if err != nil { t.Log(err) t.Fail() } assert.Equal(t, - pendingCommand.CommandInfo.Metadata.ExecutionId.String(), - testInteractionCommand.Metadata.ExecutionId.String(), + pendingCommand.CommandInfo.Metadata.RunId.String(), + testInteractionCommand.Metadata.RunId.String(), ) assert.Equal(t, pendingCommand.CommandInfo.Metadata.StepId, testInteractionCommand.Metadata.StepId, ) - err = interaction.removeInteractionFromPending(testMetadata) + err = interaction.removeCommandFromPending(testMetadata) if err != nil { t.Log(err) t.Fail() } - err = interaction.removeInteractionFromPending(testMetadata) + err = interaction.removeCommandFromPending(testMetadata) if err == nil { t.Log(err) t.Fail() } expectedErr := manualModel.ErrorPendingCommandNotFound{ - Err: "no pending commands found for execution " + + Err: "no pending commands found for run " + "61a6c41e-6efc-4516-a242-dfbc5c89d562", } assert.Equal(t, err, expectedErr) @@ -542,10 +596,12 @@ func NewTestLogHook() *TestHook { } var testUUIDStr string = "61a6c41e-6efc-4516-a242-dfbc5c89d562" -var testMetadata = execution.Metadata{ - ExecutionId: uuid.MustParse(testUUIDStr), - PlaybookId: "test_playbook_id", - StepId: "test_step_id", +var testStepRunIdStr string = "11a6c41e-6efc-4516-a242-dfbc5c89d562" +var testMetadata = run.Metadata{ + RunId: uuid.MustParse(testUUIDStr), + PlaybookId: "test_playbook_id", + StepId: "test_step_id", + StepRunId: uuid.MustParse(testStepRunIdStr), } var testInteractionCommand = manualModel.CommandInfo{ @@ -561,16 +617,18 @@ var testInteractionCommand = manualModel.CommandInfo{ }, }, Context: capability.Context{ - Command: cacao.Command{ - Type: "test_type", - Command: "test_command", - Description: "test_description", - CommandB64: "test_command_b64", - Version: "1.0", - PlaybookActivity: "test_activity", - Headers: cacao.Headers{}, - Content: "test_content", - ContentB64: "test_content_b64", + Commands: []cacao.Command{ + { + Type: "test_type", + Command: "test_command", + Description: "test_description", + CommandB64: "test_command_b64", + Version: "1.0", + PlaybookActivity: "test_activity", + Headers: cacao.Headers{}, + Content: "test_content", + ContentB64: "test_content_b64", + }, }, Step: cacao.Step{ Type: "test_type", @@ -596,12 +654,16 @@ var testInteractionCommand = manualModel.CommandInfo{ }, }, }, - Authentication: cacao.AuthenticationInformation{}, - Target: cacao.AgentTarget{ - ID: "test_id", - Type: "test_type", - Name: "test_name", - Description: "test_description", + Targets: []capability.ResolvedTarget{ + { + Authentication: cacao.AuthenticationInformation{}, + Target: cacao.AgentTarget{ + ID: "test_id", + Type: "test_type", + Name: "test_name", + Description: "test_description", + }, + }, }, Variables: cacao.Variables{ "var2": { diff --git a/pkg/core/capability/manual/manual_test.go b/internal/workflow/capability/manual/manual_test.go similarity index 57% rename from pkg/core/capability/manual/manual_test.go rename to internal/workflow/capability/manual/manual_test.go index 038fe644..64e9956e 100644 --- a/pkg/core/capability/manual/manual_test.go +++ b/internal/workflow/capability/manual/manual_test.go @@ -1,11 +1,12 @@ package manual import ( - "soarca/pkg/core/capability" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" - manualModel "soarca/pkg/models/manual" - "soarca/test/unittest/mocks/mock_interaction" + "soarca/internal/workflow/capability" + "soarca/pkg/cacao" + manualModel "soarca/internal/manual/model" + "soarca/internal/runs/model" + "soarca/pkg/utils" + "soarca/test/unittest/mocks/mock_manual_inbox" "sync" "testing" "time" @@ -14,26 +15,27 @@ import ( "github.com/stretchr/testify/mock" ) -func TestManualExecution(t *testing.T) { - interactionMock := mock_interaction.MockInteraction{} - var capturedComm manualModel.ManualCapabilityCommunication +func TestManualRun(t *testing.T) { + interactionMock := mock_manual_inbox.MockInbox{} + var capturedComm manualModel.Waiter manual := New(&interactionMock) - meta := execution.Metadata{} + meta := run.Metadata{} commandContext := capability.Context{} command := manualModel.CommandInfo{ - Metadata: execution.Metadata{}, + Metadata: run.Metadata{}, Context: capability.Context{}, OutArgsVariables: cacao.NewVariables(), } // Capture the channel passed to Queue - interactionMock.On("Queue", command, mock_interaction.AnyManualCapabilityCommunication()).Return(nil).Run(func(args mock.Arguments) { - capturedComm = args.Get(1).(manualModel.ManualCapabilityCommunication) + interactionMock.On("Queue", command, mock_manual_inbox.AnyWaiter()).Return(nil).Run(func(args mock.Arguments) { + capturedComm = args.Get(1).(manualModel.Waiter) }) + interactionMock.On("Deregister", meta).Return(nil) // Use a WaitGroup to wait for the Execute method to complete var wg sync.WaitGroup @@ -48,7 +50,7 @@ func TestManualExecution(t *testing.T) { // Simulate the response after ensuring the channel is captured time.Sleep(100 * time.Millisecond) - capturedComm.Channel <- manualModel.InteractionResponse{ + capturedComm.Channel <- manualModel.Response{ OutArgsVariables: cacao.NewVariables(), } @@ -58,14 +60,14 @@ func TestManualExecution(t *testing.T) { } func TestTimetoutCalculationNotSet(t *testing.T) { - interactionMock := mock_interaction.MockInteraction{} + interactionMock := mock_manual_inbox.MockInbox{} manual := New(&interactionMock) timeout := manual.getTimeoutValue(0) - assert.Equal(t, timeout, time.Minute) + assert.Equal(t, timeout, utils.DefaultStepTimeout()) } func TestTimetoutCalculation(t *testing.T) { - interactionMock := mock_interaction.MockInteraction{} + interactionMock := mock_manual_inbox.MockInbox{} manual := New(&interactionMock) timeout := manual.getTimeoutValue(1) assert.Equal(t, timeout, time.Millisecond*1) diff --git a/internal/workflow/capability/openc2/capability.go b/internal/workflow/capability/openc2/capability.go new file mode 100644 index 00000000..98116ff9 --- /dev/null +++ b/internal/workflow/capability/openc2/capability.go @@ -0,0 +1,85 @@ +package openc2 + +import ( + "reflect" + + "soarca/internal/logger" + "soarca/internal/workflow/capability" + "soarca/pkg/cacao" + "soarca/internal/runs/model" + "soarca/pkg/utils/http" +) + +type OpenC2Capability struct { + httpRequest http.IHttpRequest +} + +type Empty struct{} + +const ( + openc2ResultVariableName = "__soarca_openc2_http_result__" + openc2CapabilityName = "soarca-openc2-http" +) + +var ( + component = reflect.TypeOf(Empty{}).PkgPath() + log *logger.Log +) + +func init() { + log = logger.Logger(component, logger.Info, "", logger.Json) +} + +func New(httpRequest http.IHttpRequest) *OpenC2Capability { + return &OpenC2Capability{httpRequest: httpRequest} +} + +func (OpenC2Capability *OpenC2Capability) GetType() string { + return openc2CapabilityName +} + +func (OpenC2Capability *OpenC2Capability) Execute( + metadata run.Metadata, + context capability.Context, +) (cacao.Variables, error) { + log.Trace(metadata.RunId) + + // This capability performs commands against a target; a step declaring + // zero targets has nothing to run against, so skip without error. + if len(context.Targets) == 0 { + return cacao.NewVariables(), nil + } + targets := context.Targets + + returnVariables := cacao.NewVariables() + var stepErr error + + for _, resolvedTarget := range targets { + target := resolvedTarget.Target + auth := resolvedTarget.Authentication + + for _, command := range context.Commands { + httpOptions := http.HttpOptions{ + Command: &command, + Target: &target, + Auth: &auth, + } + response, err := OpenC2Capability.httpRequest.Request(httpOptions) + if err != nil { + log.Error(err) + stepErr = err + // Abort this target's remaining commands on first failure, + // but keep processing the other targets. + break + } + + results := cacao.NewVariables(cacao.Variable{Type: cacao.VariableTypeString, + Name: openc2ResultVariableName, + Value: string(response)}) + log.Trace("Finished openc2 run, will return the variables: ", results) + returnVariables.Merge(results) + } + } + + return returnVariables, stepErr +} diff --git a/pkg/core/capability/openc2/openc2_test.go b/internal/workflow/capability/openc2/openc2_test.go similarity index 52% rename from pkg/core/capability/openc2/openc2_test.go rename to internal/workflow/capability/openc2/openc2_test.go index 2f94df05..90390b01 100644 --- a/pkg/core/capability/openc2/openc2_test.go +++ b/internal/workflow/capability/openc2/openc2_test.go @@ -3,9 +3,9 @@ package openc2 import ( "testing" - "soarca/pkg/core/capability" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" + "soarca/internal/workflow/capability" + "soarca/pkg/cacao" + "soarca/internal/runs/model" "soarca/pkg/utils/http" mockRequest "soarca/test/unittest/mocks/mock_utils/http" @@ -18,7 +18,7 @@ func TestOpenC2Request(t *testing.T) { openc2 := New(mockHttp) authId, _ := uuid.Parse("6aa7b810-9dad-11d1-81b4-00c04fd430c8") - executionId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + runId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") playbookId, _ := uuid.Parse("d09351a2-a075-40c8-8054-0b7c423db83f") stepId, _ := uuid.Parse("81eff59f-d084-4324-9e0a-59e353dbd28f") @@ -47,10 +47,10 @@ func TestOpenC2Request(t *testing.T) { Value: "", } - metadata := execution.Metadata{ - ExecutionId: executionId, - PlaybookId: playbookId.String(), - StepId: stepId.String(), + metadata := run.Metadata{ + RunId: runId, + PlaybookId: playbookId.String(), + StepId: stepId.String(), } httpOptions := http.HttpOptions{ @@ -65,10 +65,11 @@ func TestOpenC2Request(t *testing.T) { mockHttp.On("Request", httpOptions).Return(payloadBytes, nil) - data := capability.Context{Command: command, - Authentication: auth, - Target: target, - Variables: cacao.NewVariables(cacaoVariable)} + data := capability.Context{ + Commands: []cacao.Command{command}, + Targets: []capability.ResolvedTarget{{Target: target, Authentication: auth}}, + Variables: cacao.NewVariables(cacaoVariable), + } results, err := openc2.Execute( metadata, @@ -80,3 +81,37 @@ func TestOpenC2Request(t *testing.T) { t.Log(results) assert.Equal(t, results["__soarca_openc2_http_result__"].Value, payload) } + +func TestOpenC2ExecuteNoTargetsSkipsWithoutError(t *testing.T) { + mockHttp := &mockRequest.MockHttpRequest{} + openc2 := New(mockHttp) + + runId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + playbookId, _ := uuid.Parse("d09351a2-a075-40c8-8054-0b7c423db83f") + stepId, _ := uuid.Parse("81eff59f-d084-4324-9e0a-59e353dbd28f") + + command := cacao.Command{ + Type: "http-api", + Command: "POST / HTTP/1.1", + } + + metadata := run.Metadata{ + RunId: runId, + PlaybookId: playbookId.String(), + StepId: stepId.String(), + } + + data := capability.Context{ + Commands: []cacao.Command{command}, + Targets: []capability.ResolvedTarget{}, + } + + results, err := openc2.Execute(metadata, data) + if err != nil { + t.Log(err) + t.Fail() + } + assert.Equal(t, len(results), 0) + + mockHttp.AssertNotCalled(t, "Request") +} diff --git a/pkg/core/capability/powershell/powershell.go b/internal/workflow/capability/powershell/capability.go similarity index 62% rename from pkg/core/capability/powershell/powershell.go rename to internal/workflow/capability/powershell/capability.go index f3093b18..4eaa7add 100644 --- a/pkg/core/capability/powershell/powershell.go +++ b/internal/workflow/capability/powershell/capability.go @@ -9,9 +9,9 @@ import ( "strings" "soarca/internal/logger" - "soarca/pkg/core/capability" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" + capabilityPkg "soarca/internal/workflow/capability" + "soarca/pkg/cacao" + "soarca/internal/runs/model" "github.com/masterzen/winrm" ) @@ -45,24 +45,56 @@ func (capability *PowershellCapability) GetType() string { } func (capability *PowershellCapability) Execute( - metadata execution.Metadata, - capabilityContext capability.Context, + metadata run.Metadata, + capabilityContext capabilityPkg.Context, ) (cacao.Variables, error) { - log.Trace(metadata.ExecutionId) + log.Trace(metadata.RunId) - port, err := strconv.Atoi(capabilityContext.Target.Port) + // This capability performs commands against a target; a step declaring + // zero targets has nothing to run against, so skip without error. + if len(capabilityContext.Targets) == 0 { + return cacao.NewVariables(), nil + } + targets := capabilityContext.Targets + + returnVariables := cacao.NewVariables() + var stepErr error + + for _, resolvedTarget := range targets { + for _, command := range capabilityContext.Commands { + results, err := capability.executeCommand(resolvedTarget.Target, resolvedTarget.Authentication, command) + returnVariables.Merge(results) + if err != nil { + log.Error(err) + stepErr = err + // Abort this target's remaining commands on first failure, + // but keep processing the other targets. + break + } + } + } + + return returnVariables, stepErr +} + +func (capability *PowershellCapability) executeCommand( + target cacao.AgentTarget, + authentication cacao.AuthenticationInformation, + command cacao.Command, +) (cacao.Variables, error) { + port, err := strconv.Atoi(target.Port) if err != nil { log.Error("port is not parsable " + err.Error()) return cacao.NewVariables(), err } - address, err := determineTargetAddress(capabilityContext.Target) + address, err := determineTargetAddress(target) if err != nil { return cacao.NewVariables(), err } endpoint := winrm.NewEndpoint(address, port, false, false, nil, nil, nil, 0) - client, err := winrm.NewClient(endpoint, capabilityContext.Authentication.Username, capabilityContext.Authentication.Password) + client, err := winrm.NewClient(endpoint, authentication.Username, authentication.Password) if err != nil { log.Error("failed to create client") log.Error(err) @@ -73,14 +105,14 @@ func (capability *PowershellCapability) Execute( defer cancel() effectiveCommand := "" - if capabilityContext.Command.CommandB64 != "" { - bytes, err := base64.StdEncoding.DecodeString(capabilityContext.Command.CommandB64) + if command.CommandB64 != "" { + bytes, err := base64.StdEncoding.DecodeString(command.CommandB64) if err != nil { return cacao.NewVariables(), err } effectiveCommand = string(bytes) } else { - effectiveCommand = capabilityContext.Command.Command + effectiveCommand = command.Command } result, stdErr, _, err := client.RunPSWithContext(ctx, effectiveCommand) diff --git a/internal/workflow/capability/powershell/powershell_test.go b/internal/workflow/capability/powershell/powershell_test.go new file mode 100644 index 00000000..1f1e475f --- /dev/null +++ b/internal/workflow/capability/powershell/powershell_test.go @@ -0,0 +1,39 @@ +package powershell + +import ( + "soarca/internal/workflow/capability" + "soarca/pkg/cacao" + "soarca/internal/runs/model" + "testing" + + "github.com/go-playground/assert/v2" + "github.com/google/uuid" +) + +func TestPowershellExecuteNoTargetsSkipsWithoutError(t *testing.T) { + powershellCapability := New() + + runId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + playbookId, _ := uuid.Parse("d09351a2-a075-40c8-8054-0b7c423db83f") + stepId, _ := uuid.Parse("81eff59f-d084-4324-9e0a-59e353dbd28f") + + command := cacao.Command{Type: "powershell", Command: "Get-Process"} + + metadata := run.Metadata{ + RunId: runId, + PlaybookId: playbookId.String(), + StepId: stepId.String(), + } + + data := capability.Context{ + Commands: []cacao.Command{command}, + Targets: []capability.ResolvedTarget{}, + } + + results, err := powershellCapability.Execute(metadata, data) + if err != nil { + t.Log(err) + t.Fail() + } + assert.Equal(t, len(results), 0) +} diff --git a/pkg/core/capability/ssh/ssh.go b/internal/workflow/capability/ssh/capability.go similarity index 79% rename from pkg/core/capability/ssh/ssh.go rename to internal/workflow/capability/ssh/capability.go index 25f3498a..bd971cc1 100644 --- a/pkg/core/capability/ssh/ssh.go +++ b/internal/workflow/capability/ssh/capability.go @@ -3,9 +3,9 @@ package ssh import ( "errors" "reflect" - "soarca/pkg/core/capability" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" + "soarca/internal/workflow/capability" + "soarca/pkg/cacao" + "soarca/internal/runs/model" "strings" "time" @@ -33,11 +33,36 @@ func (sshCapability *SshCapability) GetType() string { return sshCapabilityName } -func (sshCapability *SshCapability) Execute(metadata execution.Metadata, +func (sshCapability *SshCapability) Execute(metadata run.Metadata, context capability.Context) (cacao.Variables, error) { - log.Trace(metadata.ExecutionId) - return execute(context.Command, context.Authentication, context.Target) + log.Trace(metadata.RunId) + + // This capability performs commands against a target; a step declaring + // zero targets has nothing to run against, so skip without error. + if len(context.Targets) == 0 { + return cacao.NewVariables(), nil + } + targets := context.Targets + + returnVariables := cacao.NewVariables() + var stepErr error + + for _, resolvedTarget := range targets { + for _, command := range context.Commands { + results, err := execute(command, resolvedTarget.Authentication, resolvedTarget.Target) + returnVariables.Merge(results) + if err != nil { + log.Error(err) + stepErr = err + // Abort this target's remaining commands on first failure, + // but keep processing the other targets. + break + } + } + } + + return returnVariables, stepErr } func execute(command cacao.Command, @@ -75,7 +100,7 @@ func executeCommand(session *ssh.Session, results := cacao.NewVariables(cacao.Variable{Type: cacao.VariableTypeString, Name: sshResultVariableName, Value: string(response)}) - log.Trace("Finished ssh execution will return the variables: ", results) + log.Trace("Finished ssh run will return the variables: ", results) sessionErr := session.Close() if sessionErr != nil { log.Error(sessionErr) diff --git a/pkg/core/capability/ssh/ssh_test.go b/internal/workflow/capability/ssh/ssh_test.go similarity index 81% rename from pkg/core/capability/ssh/ssh_test.go rename to internal/workflow/capability/ssh/ssh_test.go index f5aeb5aa..a2624391 100644 --- a/pkg/core/capability/ssh/ssh_test.go +++ b/internal/workflow/capability/ssh/ssh_test.go @@ -2,10 +2,13 @@ package ssh import ( "errors" - "soarca/pkg/models/cacao" + "soarca/internal/workflow/capability" + "soarca/pkg/cacao" + "soarca/internal/runs/model" "testing" "github.com/go-playground/assert/v2" + "github.com/google/uuid" ) func TestStripSshPrependWithPrepend(t *testing.T) { @@ -103,3 +106,31 @@ func TestAddressAndPortCombinationNoIpv4Address(t *testing.T) { result := CombinePortAndAddress(ipv4, port) assert.Equal(t, result, expectedFqdn) } + +func TestSshExecuteNoTargetsSkipsWithoutError(t *testing.T) { + sshCapability := &SshCapability{} + + runId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + playbookId, _ := uuid.Parse("d09351a2-a075-40c8-8054-0b7c423db83f") + stepId, _ := uuid.Parse("81eff59f-d084-4324-9e0a-59e353dbd28f") + + command := cacao.Command{Type: "ssh", Command: "ls -la"} + + metadata := run.Metadata{ + RunId: runId, + PlaybookId: playbookId.String(), + StepId: stepId.String(), + } + + data := capability.Context{ + Commands: []cacao.Command{command}, + Targets: []capability.ResolvedTarget{}, + } + + results, err := sshCapability.Execute(metadata, data) + if err != nil { + t.Log(err) + t.Fail() + } + assert.Equal(t, len(results), 0) +} diff --git a/pkg/core/executors/action/action_executor_test.go b/internal/workflow/steps/action/action_executor_test.go similarity index 62% rename from pkg/core/executors/action/action_executor_test.go rename to internal/workflow/steps/action/action_executor_test.go index b816d826..63554827 100644 --- a/pkg/core/executors/action/action_executor_test.go +++ b/internal/workflow/steps/action/action_executor_test.go @@ -5,10 +5,10 @@ import ( "testing" "time" - "soarca/pkg/core/capability" - "soarca/pkg/core/executors" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" + "soarca/internal/workflow/capability" + "soarca/internal/workflow/steps" + "soarca/pkg/cacao" + "soarca/internal/runs/model" "soarca/test/unittest/mocks/mock_assignment_extension" "soarca/test/unittest/mocks/mock_capability" "soarca/test/unittest/mocks/mock_reporter" @@ -25,14 +25,14 @@ func TestExecuteStep(t *testing.T) { mock_time := new(mock_time.MockTime) mock_assignment := new(mock_assignment_extension.Mock_AssignmentExtension) - capabilities := map[string]capability.ICapability{"mock-ssh": mock_ssh, "http-api": mock_http} + capabilities := map[string]capability.ICapability{"ssh": mock_ssh, "http-api": mock_http} executerObject := New(capabilities, mock_reporter, mock_time, mock_assignment) - executionId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + runId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") playbookId := "playbook--d09351a2-a075-40c8-8054-0b7c423db83f" stepId := "step--81eff59f-d084-4324-9e0a-59e353dbd28f" - metadata := execution.Metadata{ExecutionId: executionId, PlaybookId: playbookId, StepId: stepId} + metadata := run.Metadata{RunId: runId, PlaybookId: playbookId, StepId: stepId} expectedCommand := cacao.Command{ Type: "ssh", @@ -81,11 +81,11 @@ func TestExecuteStep(t *testing.T) { } context1 := capability.Context{ - Command: expectedCommand, - Authentication: expectedAuth, - Target: expectedTarget, - Variables: cacao.NewVariables(expectedVariables), - Step: step, + Commands: []cacao.Command{expectedCommand}, + Targets: []capability.ResolvedTarget{{Target: expectedTarget, Authentication: expectedAuth}}, + Variables: cacao.NewVariables(expectedVariables), + Step: step, + Agent: agent, } layout := "2006-01-02T15:04:05.000Z" @@ -93,9 +93,9 @@ func TestExecuteStep(t *testing.T) { timeNow, _ := time.Parse(layout, str) mock_time.On("Now").Return(timeNow) - mock_reporter.On("ReportStepStart", executionId, step, cacao.NewVariables(expectedVariables), timeNow).Return() + mock_reporter.On("ReportStepStart", metadata, step, cacao.NewVariables(expectedVariables), timeNow).Return() - mock_reporter.On("ReportStepEnd", executionId, step, cacao.NewVariables(expectedVariables), nil, timeNow).Return() + mock_reporter.On("ReportStepEnd", metadata, step, cacao.NewVariables(expectedVariables), nil, timeNow).Return() mock_ssh.On("Execute", metadata, context1). @@ -121,11 +121,11 @@ func TestExecuteActionStep(t *testing.T) { capabilities := map[string]capability.ICapability{"ssh": mock_ssh, "http-api": mock_http} executerObject := New(capabilities, mock_reporter, mock_time, mock_assignment) - executionId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + runId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") playbookId := "playbook--d09351a2-a075-40c8-8054-0b7c423db83f" stepId := "step--81eff59f-d084-4324-9e0a-59e353dbd28f" - metadata := execution.Metadata{ExecutionId: executionId, PlaybookId: playbookId, StepId: stepId} + metadata := run.Metadata{RunId: runId, PlaybookId: playbookId, StepId: stepId} expectedCommand := cacao.Command{ Type: "ssh", @@ -152,10 +152,10 @@ func TestExecuteActionStep(t *testing.T) { } context1 := capability.Context{ - Command: expectedCommand, - Authentication: expectedAuth, - Target: expectedTarget, - Variables: cacao.NewVariables(expectedVariables), + Commands: []cacao.Command{expectedCommand}, + Targets: []capability.ResolvedTarget{{Target: expectedTarget, Authentication: expectedAuth}}, + Variables: cacao.NewVariables(expectedVariables), + Agent: agent, } mock_ssh.On("Execute", @@ -164,11 +164,10 @@ func TestExecuteActionStep(t *testing.T) { Return(cacao.NewVariables(expectedVariables), nil) - data := data{command: expectedCommand, - authentication: expectedAuth, - target: expectedTarget, - variables: cacao.NewVariables(expectedVariables), - agent: agent} + data := data{commands: []cacao.Command{expectedCommand}, + targets: []capability.ResolvedTarget{{Target: expectedTarget, Authentication: expectedAuth}}, + variables: cacao.NewVariables(expectedVariables), + agent: agent} _, err := executerObject.executeCommands(metadata, data) @@ -188,11 +187,11 @@ func TestNonExistingCapabilityStep(t *testing.T) { capabilities := map[string]capability.ICapability{"ssh": mock_ssh, "http-api": mock_http} executerObject := New(capabilities, new(mock_reporter.Mock_Reporter), mock_time, mock_assignment) - executionId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + runId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") playbookId := "playbook--d09351a2-a075-40c8-8054-0b7c423db83f" stepId := "step--81eff59f-d084-4324-9e0a-59e353dbd28f" - metadata := execution.Metadata{ExecutionId: executionId, PlaybookId: playbookId, StepId: stepId} + metadata := run.Metadata{RunId: runId, PlaybookId: playbookId, StepId: stepId} expectedCommand := cacao.Command{ Type: "ssh", @@ -214,15 +213,14 @@ func TestNonExistingCapabilityStep(t *testing.T) { } agent := cacao.AgentTarget{ - Type: "ssh", + Type: "non-existing", Name: "non-existing", } - data := data{command: expectedCommand, - authentication: expectedAuth, - target: expectedTarget, - variables: cacao.NewVariables(expectedVariables), - agent: agent} + data := data{commands: []cacao.Command{expectedCommand}, + targets: []capability.ResolvedTarget{{Target: expectedTarget, Authentication: expectedAuth}}, + variables: cacao.NewVariables(expectedVariables), + agent: agent} _, err := executerObject.executeCommands(metadata, data) @@ -231,6 +229,72 @@ func TestNonExistingCapabilityStep(t *testing.T) { mock_time.AssertExpectations(t) } +func TestUnknownCapabilityTypeRoutesToFinFallbackWhenConfigured(t *testing.T) { + mock_ssh := new(mock_capability.Mock_Capability) + mock_fin := new(mock_capability.Mock_Capability) + mock_time := new(mock_time.MockTime) + mock_assignment := new(mock_assignment_extension.Mock_AssignmentExtension) + + capabilities := map[string]capability.ICapability{"ssh": mock_ssh} + + executerObject := New(capabilities, new(mock_reporter.Mock_Reporter), mock_time, mock_assignment) + executerObject.SetFinFallback(mock_fin) + + runId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + playbookId := "playbook--d09351a2-a075-40c8-8054-0b7c423db83f" + stepId := "step--81eff59f-d084-4324-9e0a-59e353dbd28f" + + metadata := run.Metadata{RunId: runId, PlaybookId: playbookId, StepId: stepId} + + expectedCommand := cacao.Command{ + Type: "http-executor", + Command: "do the thing", + } + + expectedVariables := cacao.Variable{ + Type: "string", + Name: "var1", + Value: "testing", + } + + expectedAuth := cacao.AuthenticationInformation{ + Name: "user", + } + + expectedTarget := cacao.AgentTarget{ + Name: "sometarget", + } + + // A capability type unknown to the static capabilities map must not + // error out - it must fall through to the Fin fallback capability, + // which is exactly how dynamically-registered Fin capability types get + // routed (see Executor.SetFinFallback). + agent := cacao.AgentTarget{ + Type: "some-fin-capability", + Name: "some fin", + } + + expectedContext := capability.Context{ + Commands: []cacao.Command{expectedCommand}, + Targets: []capability.ResolvedTarget{{Target: expectedTarget, Authentication: expectedAuth}}, + Variables: cacao.NewVariables(expectedVariables), + Agent: agent, + } + + mock_fin.On("Execute", metadata, expectedContext).Return(cacao.NewVariables(expectedVariables), nil) + + data := data{commands: []cacao.Command{expectedCommand}, + targets: []capability.ResolvedTarget{{Target: expectedTarget, Authentication: expectedAuth}}, + variables: cacao.NewVariables(expectedVariables), + agent: agent} + _, err := executerObject.executeCommands(metadata, data) + + assert.Equal(t, err, nil) + mock_fin.AssertExpectations(t) + mock_ssh.AssertExpectations(t) + mock_time.AssertExpectations(t) +} + func TestVariableInterpolation(t *testing.T) { mock_capability1 := new(mock_capability.Mock_Capability) mock_time := new(mock_time.MockTime) @@ -239,11 +303,11 @@ func TestVariableInterpolation(t *testing.T) { capabilities := map[string]capability.ICapability{"cap1": mock_capability1} executerObject := New(capabilities, new(mock_reporter.Mock_Reporter), mock_time, mock_assignment) - executionId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + runId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") playbookId := "playbook--d09351a2-a075-40c8-8054-0b7c423db83f" stepId := "step--81eff59f-d084-4324-9e0a-59e353dbd28f" - metadata := execution.Metadata{ExecutionId: executionId, PlaybookId: playbookId, StepId: stepId} + metadata := run.Metadata{RunId: runId, PlaybookId: playbookId, StepId: stepId} inputCommand := cacao.Command{ Type: "ssh", @@ -368,14 +432,14 @@ func TestVariableInterpolation(t *testing.T) { } agent := cacao.AgentTarget{ - Type: "ssh", + Type: "cap1", Name: "cap1", } - context1 := capability.Context{Command: expectedCommand, - Authentication: expectedAuth, - Target: expectedTarget, - Variables: cacao.NewVariables(var1, var2, var3, varUser, varPassword, varOauth, varPrivateKey, varToken, varUserId, varheader1, varheader2)} + context1 := capability.Context{Commands: []cacao.Command{expectedCommand}, + Targets: []capability.ResolvedTarget{{Target: expectedTarget, Authentication: expectedAuth}}, + Variables: cacao.NewVariables(var1, var2, var3, varUser, varPassword, varOauth, varPrivateKey, varToken, varUserId, varheader1, varheader2), + Agent: agent} mock_capability1.On("Execute", metadata, @@ -383,11 +447,10 @@ func TestVariableInterpolation(t *testing.T) { Return(cacao.NewVariables(var1), nil) - data1 := data{command: inputCommand, - authentication: inputAuth, - target: inputTarget, - variables: cacao.NewVariables(var1, var2, var3, varUser, varPassword, varOauth, varPrivateKey, varToken, varUserId, varheader1, varheader2), - agent: agent} + data1 := data{commands: []cacao.Command{inputCommand}, + targets: []capability.ResolvedTarget{{Target: inputTarget, Authentication: inputAuth}}, + variables: cacao.NewVariables(var1, var2, var3, varUser, varPassword, varOauth, varPrivateKey, varToken, varUserId, varheader1, varheader2), + agent: agent} _, err := executerObject.executeCommands(metadata, data1) @@ -412,11 +475,11 @@ func TestVariableInterpolation(t *testing.T) { Headers: expectedHeaders, } - metadataHttp := execution.Metadata{ExecutionId: executionId, PlaybookId: playbookId, StepId: stepId} - contextHttp := capability.Context{Command: expectedHttpCommand, - Authentication: expectedAuth, - Target: expectedTarget, - Variables: cacao.NewVariables(varHttpContent, varheader1, varheader2)} + metadataHttp := run.Metadata{RunId: runId, PlaybookId: playbookId, StepId: stepId} + contextHttp := capability.Context{Commands: []cacao.Command{expectedHttpCommand}, + Targets: []capability.ResolvedTarget{{Target: expectedTarget, Authentication: expectedAuth}}, + Variables: cacao.NewVariables(varHttpContent, varheader1, varheader2), + Agent: agent} mock_capability1.On("Execute", metadataHttp, @@ -424,11 +487,10 @@ func TestVariableInterpolation(t *testing.T) { Return(cacao.NewVariables(var1), nil) - data2 := data{command: httpCommand, - authentication: expectedAuth, - target: expectedTarget, - variables: cacao.NewVariables(varHttpContent, varheader1, varheader2), - agent: agent} + data2 := data{commands: []cacao.Command{httpCommand}, + targets: []capability.ResolvedTarget{{Target: expectedTarget, Authentication: expectedAuth}}, + variables: cacao.NewVariables(varHttpContent, varheader1, varheader2), + agent: agent} _, err = executerObject.executeCommands(metadata, data2) diff --git a/internal/workflow/steps/action/executor.go b/internal/workflow/steps/action/executor.go new file mode 100644 index 00000000..f74d556e --- /dev/null +++ b/internal/workflow/steps/action/executor.go @@ -0,0 +1,236 @@ +package action + +import ( + "errors" + "fmt" + "reflect" + "soarca/internal/logger" + "soarca/internal/reporting/reporter" + run "soarca/internal/runs/model" + "soarca/internal/workflow/capability" + executors "soarca/internal/workflow/steps" + "soarca/pkg/cacao" + "soarca/pkg/extensions/soarca/assignment" + timeUtil "soarca/pkg/utils/time" +) + +var component = reflect.TypeOf(Executor{}).PkgPath() +var log *logger.Log + +func init() { + log = logger.Logger(component, logger.Info, "", logger.Json) +} + +func New(capabilities map[string]capability.ICapability, reporter reporter.IStepReporter, time timeUtil.ITime, assigner assignment.IAssignmentExtension) *Executor { + var instance = Executor{} + instance.capabilities = capabilities + instance.reporter = reporter + instance.time = time + instance.assigner = assigner + return &instance +} + +type IExecuter interface { + Execute(metadata run.Metadata, + step executors.PlaybookStepMetadata) (cacao.Variables, error) +} + +type Executor struct { + capabilities map[string]capability.ICapability + // finFallback is consulted whenever data.agent.Type doesn't match any + // statically-registered capability. Unlike built-ins, Fin capability + // types are declared dynamically at Fin registration time and can't be + // known up front to populate capabilities, so a single fallback + // capability (routing internally on Context.Agent.Type) stands in for + // the whole, open-ended set of them. Nil if Fin support is disabled. + finFallback capability.ICapability + reporter reporter.IStepReporter + time timeUtil.ITime + assigner assignment.IAssignmentExtension +} + +// SetFinFallback wires the fallback capability consulted for any +// agent.Type not present in the static capabilities map (see +// Executor.finFallback). Passing nil disables Fin routing entirely. +func (executor *Executor) SetFinFallback(finFallback capability.ICapability) { + executor.finFallback = finFallback +} + +type data struct { + commands []cacao.Command + targets []capability.ResolvedTarget + variables cacao.Variables + agent cacao.AgentTarget + step cacao.Step +} + +func (executor *Executor) Execute(meta run.Metadata, + metadata executors.PlaybookStepMetadata) (cacao.Variables, error) { + + executor.reporter.ReportStepStart(meta, metadata.Step, metadata.Variables, executor.time.Now()) + + returnVariables := cacao.NewVariables() + var err error + defer func() { + executor.reporter.ReportStepEnd(meta, metadata.Step, returnVariables, err, executor.time.Now()) + }() + + if metadata.Step.Type != cacao.StepTypeAction { + err = errors.New("the provided step type is not compatible with this executor") + log.Error(err) + return cacao.NewVariables(), err + } + + returnVariables, err = executor.executeCommandFromArray(meta, metadata) + return returnVariables, err +} + +func (executor *Executor) executeCommandFromArray(meta run.Metadata, + metadata executors.PlaybookStepMetadata) (cacao.Variables, error) { + + // NOTE: interpolation happens once per command/target below, inside + // executeCommands, so raw (uninterpolated) values are passed through + // here; only the target->auth resolution needs the raw step data. + // + // NOTE: an action step's targets are optional per the CACAO spec — an + // empty Targets list is a valid shape, not a signal to skip run. + // What about Agent authentication? (left as a pre-existing open question) + targets := make([]capability.ResolvedTarget, 0, len(metadata.Step.Targets)) + for _, element := range metadata.Step.Targets { + target := metadata.Targets[element] + auth := metadata.Auth[target.AuthInfoIdentifier] + targets = append(targets, capability.ResolvedTarget{ + Target: target, + Authentication: auth, + }) + } + + data := data{ + commands: metadata.Step.Commands, + targets: targets, + variables: metadata.Variables, + agent: metadata.Agent, + step: metadata.Step, + } + + outputVariables, err := executor.executeCommands(meta, data) + if err != nil { + log.Error("Error executing Command ", err) + return cacao.NewVariables(), err + } + log.Trace("Command executed") + + // Map defined step results into variables as described by any + // soarca-assignment step extensions. + assignedVariables := executor.evaluateAssignments(metadata.Step.StepExtensions, outputVariables) + outputVariables.Merge(assignedVariables) + + if len(metadata.Step.OutArgs) > 0 { + // If OutArgs is set, only update run args that are explicitly referenced + outputVariables = outputVariables.Select(metadata.Step.OutArgs) + } + + return outputVariables, nil +} + +func (executor *Executor) evaluateAssignments(extensions cacao.Extensions, results cacao.Variables) cacao.Variables { + assigned := cacao.NewVariables() + for id, raw := range extensions { + switch raw.(type) { + case assignment.Assignment: + + } + model, ok := assignment.DecodeAssignment(raw) + if !ok { + continue + } + log.Trace("evaluating assignment extension ", id) + produced := executor.assigner.AssignAndEvaluate(assignment.Context{ + AssignmentModel: model, + Source: results, + }) + assigned.Merge(produced) + } + return assigned +} + +func interpolateCommand(command cacao.Command, variables cacao.Variables) cacao.Command { + command.Command = variables.Interpolate(command.Command) + command.Content = variables.Interpolate(command.Content) + command.ContentB64 = variables.Interpolate(command.ContentB64) + for key, headers := range command.Headers { + var slice []string + for _, header := range headers { + slice = append(slice, variables.Interpolate(header)) + } + command.Headers[key] = slice + } + return command +} + +func interpolatedTarget(target cacao.AgentTarget, variables cacao.Variables) cacao.AgentTarget { + for key, addresses := range target.Address { + var slice []string + for _, address := range addresses { + slice = append(slice, variables.Interpolate(address)) + } + target.Address[key] = slice + } + return target +} + +func interpolateAuthentication(authentication cacao.AuthenticationInformation, variables cacao.Variables) cacao.AuthenticationInformation { + authentication.Username = variables.Interpolate(authentication.Username) + authentication.Password = variables.Interpolate(authentication.Password) + authentication.UserId = variables.Interpolate(authentication.UserId) + authentication.Token = variables.Interpolate(authentication.Token) + authentication.OauthHeader = variables.Interpolate(authentication.OauthHeader) + authentication.PrivateKey = variables.Interpolate(authentication.PrivateKey) + + return authentication + +} + +func (executor *Executor) executeCommands(metadata run.Metadata, + data data) (cacao.Variables, error) { + + cap, ok := executor.capabilities[data.agent.Type] + if !ok { + if executor.finFallback == nil { + empty := cacao.NewVariables() + err := errors.New(fmt.Sprint("capability: ", data.agent.Type, " is not available in soarca")) + log.Error(err) + return empty, err + } + cap = executor.finFallback + } + + context := capability.Context{ + Commands: interpolateCommands(data.commands, data.variables), + Targets: interpolateTargets(data.targets, data.variables), + Variables: data.variables, + Step: data.step, + Agent: data.agent, + } + returnVariables, err := cap.Execute(metadata, context) + return returnVariables, err +} + +func interpolateCommands(commands []cacao.Command, variables cacao.Variables) []cacao.Command { + interpolated := make([]cacao.Command, 0, len(commands)) + for _, command := range commands { + interpolated = append(interpolated, interpolateCommand(command, variables)) + } + return interpolated +} + +func interpolateTargets(targets []capability.ResolvedTarget, variables cacao.Variables) []capability.ResolvedTarget { + interpolated := make([]capability.ResolvedTarget, 0, len(targets)) + for _, resolvedTarget := range targets { + interpolated = append(interpolated, capability.ResolvedTarget{ + Target: interpolatedTarget(resolvedTarget.Target, variables), + Authentication: interpolateAuthentication(resolvedTarget.Authentication, variables), + }) + } + return interpolated +} diff --git a/pkg/core/executors/condition/condition_executor_test.go b/internal/workflow/steps/condition/condition_executor_test.go similarity index 80% rename from pkg/core/executors/condition/condition_executor_test.go rename to internal/workflow/steps/condition/condition_executor_test.go index ce0a3cff..2506bbc3 100644 --- a/pkg/core/executors/condition/condition_executor_test.go +++ b/internal/workflow/steps/condition/condition_executor_test.go @@ -2,9 +2,9 @@ package condition import ( "errors" - "soarca/pkg/core/executors" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" + "soarca/internal/workflow/steps" + "soarca/pkg/cacao" + "soarca/internal/runs/model" "soarca/test/unittest/mocks/mock_reporter" mock_stix "soarca/test/unittest/mocks/mock_utils/stix" mock_time "soarca/test/unittest/mocks/mock_utils/time" @@ -22,9 +22,9 @@ func TestExecuteConditionTrue(t *testing.T) { conditionExecutior := New(mock_stix, mock_reporter, mock_time) - executionId := uuid.New() + runId := uuid.New() - meta := execution.Metadata{ExecutionId: executionId, + meta := run.Metadata{RunId: runId, PlaybookId: "1", StepId: "2"} @@ -39,9 +39,9 @@ func TestExecuteConditionTrue(t *testing.T) { timeNow, _ := time.Parse(layout, str) mock_time.On("Now").Return(timeNow) - mock_reporter.On("ReportStepStart", executionId, step, vars, timeNow) + mock_reporter.On("ReportStepStart", meta, step, vars, timeNow) mock_stix.On("Evaluate", "a = a", vars).Return(true, nil) - mock_reporter.On("ReportStepEnd", executionId, step, vars, nil, timeNow) + mock_reporter.On("ReportStepEnd", meta, step, vars, nil, timeNow) context := executors.Context{Step: step, Variables: vars} nextStepId, goToBranch, err := conditionExecutior.Execute(meta, context) assert.Equal(t, nil, err) @@ -60,9 +60,9 @@ func TestExecuteConditionFalse(t *testing.T) { conditionExecutior := New(mock_stix, mock_reporter, mock_time) - executionId := uuid.New() + runId := uuid.New() - meta := execution.Metadata{ExecutionId: executionId, + meta := run.Metadata{RunId: runId, PlaybookId: "1", StepId: "2"} @@ -77,9 +77,9 @@ func TestExecuteConditionFalse(t *testing.T) { timeNow, _ := time.Parse(layout, str) mock_time.On("Now").Return(timeNow) - mock_reporter.On("ReportStepStart", executionId, step, vars, timeNow) + mock_reporter.On("ReportStepStart", meta, step, vars, timeNow) mock_stix.On("Evaluate", "a = a", vars).Return(false, nil) - mock_reporter.On("ReportStepEnd", executionId, step, vars, nil, timeNow) + mock_reporter.On("ReportStepEnd", meta, step, vars, nil, timeNow) context := executors.Context{Step: step, Variables: vars} nextStepId, goToBranch, err := conditionExecutior.Execute(meta, context) @@ -99,9 +99,9 @@ func TestExecuteConditionError(t *testing.T) { conditionExecutior := New(mock_stix, mock_reporter, mock_time) - executionId := uuid.New() + runId := uuid.New() - meta := execution.Metadata{ExecutionId: executionId, + meta := run.Metadata{RunId: runId, PlaybookId: "1", StepId: "2"} @@ -118,9 +118,9 @@ func TestExecuteConditionError(t *testing.T) { evaluationError := errors.New("some ds error") - mock_reporter.On("ReportStepStart", executionId, step, vars, timeNow).Return() + mock_reporter.On("ReportStepStart", meta, step, vars, timeNow).Return() mock_stix.On("Evaluate", "a = a", vars).Return(false, evaluationError) - mock_reporter.On("ReportStepEnd", executionId, step, vars, evaluationError, timeNow).Return() + mock_reporter.On("ReportStepEnd", meta, step, vars, evaluationError, timeNow).Return() context := executors.Context{Step: step, Variables: vars} nextStepId, goToBranch, err := conditionExecutior.Execute(meta, context) @@ -140,9 +140,9 @@ func TestExecuteConditionWhile(t *testing.T) { conditionExecutior := New(mock_stix, mock_reporter, mock_time) - executionId := uuid.New() + runId := uuid.New() - meta := execution.Metadata{ExecutionId: executionId, + meta := run.Metadata{RunId: runId, PlaybookId: "1", StepId: "2"} @@ -157,9 +157,9 @@ func TestExecuteConditionWhile(t *testing.T) { timeNow, _ := time.Parse(layout, str) mock_time.On("Now").Return(timeNow) - mock_reporter.On("ReportStepStart", executionId, step, vars, timeNow) + mock_reporter.On("ReportStepStart", meta, step, vars, timeNow) mock_stix.On("Evaluate", "a = a", vars).Return(true, nil) - mock_reporter.On("ReportStepEnd", executionId, step, vars, nil, timeNow) + mock_reporter.On("ReportStepEnd", meta, step, vars, nil, timeNow) context := executors.Context{Step: step, Variables: vars} nextStepId, goToBranch, err := conditionExecutior.Execute(meta, context) @@ -179,9 +179,9 @@ func TestExecuteConditionWhileFalse(t *testing.T) { conditionExecutior := New(mock_stix, mock_reporter, mock_time) - executionId := uuid.New() + runId := uuid.New() - meta := execution.Metadata{ExecutionId: executionId, + meta := run.Metadata{RunId: runId, PlaybookId: "1", StepId: "2"} @@ -196,9 +196,9 @@ func TestExecuteConditionWhileFalse(t *testing.T) { timeNow, _ := time.Parse(layout, str) mock_time.On("Now").Return(timeNow) - mock_reporter.On("ReportStepStart", executionId, step, vars, timeNow) + mock_reporter.On("ReportStepStart", meta, step, vars, timeNow) mock_stix.On("Evaluate", "a = b", vars).Return(false, nil) - mock_reporter.On("ReportStepEnd", executionId, step, vars, nil, timeNow) + mock_reporter.On("ReportStepEnd", meta, step, vars, nil, timeNow) context := executors.Context{Step: step, Variables: vars} nextStepId, goToBranch, err := conditionExecutior.Execute(meta, context) diff --git a/pkg/core/executors/condition/condition.go b/internal/workflow/steps/condition/evaluator.go similarity index 78% rename from pkg/core/executors/condition/condition.go rename to internal/workflow/steps/condition/evaluator.go index 5bcdb4d7..aeece2b7 100644 --- a/pkg/core/executors/condition/condition.go +++ b/internal/workflow/steps/condition/evaluator.go @@ -5,10 +5,10 @@ import ( "fmt" "reflect" "soarca/internal/logger" - "soarca/pkg/core/executors" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" - "soarca/pkg/reporting/reporter" + "soarca/internal/workflow/steps" + "soarca/pkg/cacao" + "soarca/internal/runs/model" + "soarca/internal/reporting/reporter" "soarca/pkg/utils/stix/expression/comparison" timeUtil "soarca/pkg/utils/time" ) @@ -27,7 +27,7 @@ func New(comparison comparison.IComparison, } type IExecuter interface { - Execute(metadata execution.Metadata, + Execute(metadata run.Metadata, step cacao.Step, variables cacao.Variables) (string, bool, error) } @@ -37,7 +37,7 @@ type Executor struct { time timeUtil.ITime } -func (executor *Executor) Execute(meta execution.Metadata, stepContext executors.Context) (string, bool, error) { +func (executor *Executor) Execute(meta run.Metadata, stepContext executors.Context) (string, bool, error) { if stepContext.Step.Type != cacao.StepTypeIfCondition && stepContext.Step.Type != cacao.StepTypeWhileCondition { err := errors.New("the provided step type is not compatible with this executor") @@ -45,11 +45,11 @@ func (executor *Executor) Execute(meta execution.Metadata, stepContext executors return stepContext.Step.OnFailure, false, err } - executor.reporter.ReportStepStart(meta.ExecutionId, stepContext.Step, stepContext.Variables, executor.time.Now()) + executor.reporter.ReportStepStart(meta, stepContext.Step, stepContext.Variables, executor.time.Now()) var err error defer func() { - executor.reporter.ReportStepEnd(meta.ExecutionId, stepContext.Step, stepContext.Variables, err, executor.time.Now()) + executor.reporter.ReportStepEnd(meta, stepContext.Step, stepContext.Variables, err, executor.time.Now()) }() nextStepId, branch, err := executor.evaluate(stepContext) return nextStepId, branch, err diff --git a/pkg/core/executors/executors.go b/internal/workflow/steps/contracts.go similarity index 78% rename from pkg/core/executors/executors.go rename to internal/workflow/steps/contracts.go index 4b9b0454..b25b1cbd 100644 --- a/pkg/core/executors/executors.go +++ b/internal/workflow/steps/contracts.go @@ -1,12 +1,12 @@ package executors import ( - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" + "soarca/pkg/cacao" + "soarca/internal/runs/model" ) type IPlaybookExecuter interface { - Execute(execution.Metadata, + Execute(run.Metadata, cacao.Step, cacao.Variables) (cacao.Variables, error) } @@ -17,7 +17,7 @@ type Context struct { } type IConditionExecuter interface { - Execute(metadata execution.Metadata, + Execute(metadata run.Metadata, stepContext Context) (string, bool, error) } @@ -30,6 +30,6 @@ type PlaybookStepMetadata struct { } type IActionExecutor interface { - Execute(metadata execution.Metadata, + Execute(metadata run.Metadata, step PlaybookStepMetadata) (cacao.Variables, error) } diff --git a/pkg/core/executors/playbook_action/playbook_action_executor_test.go b/internal/workflow/steps/playbook_action/playbook_action_executor_test.go similarity index 59% rename from pkg/core/executors/playbook_action/playbook_action_executor_test.go rename to internal/workflow/steps/playbook_action/playbook_action_executor_test.go index cf0c9398..f5417fc4 100644 --- a/pkg/core/executors/playbook_action/playbook_action_executor_test.go +++ b/internal/workflow/steps/playbook_action/playbook_action_executor_test.go @@ -4,37 +4,35 @@ import ( "testing" "time" - "soarca/pkg/core/decomposer" - mock_database_controller "soarca/test/unittest/mocks/mock_controller/database" - mock_decomposer_controller "soarca/test/unittest/mocks/mock_controller/decomposer" - "soarca/test/unittest/mocks/mock_decomposer" + "soarca/internal/workflow" mocks_playbook_test "soarca/test/unittest/mocks/mock_playbook_database" "soarca/test/unittest/mocks/mock_reporter" mock_time "soarca/test/unittest/mocks/mock_utils/time" + "soarca/test/unittest/mocks/mock_walker" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" + "soarca/pkg/cacao" + "soarca/internal/runs/model" "github.com/go-playground/assert/v2" "github.com/google/uuid" + "github.com/stretchr/testify/mock" ) func TestExecutePlaybook(t *testing.T) { playbookRepoMock := new(mocks_playbook_test.MockPlaybook) - mockDecomposer := new(mock_decomposer.Mock_Decomposer) + mockWalker := new(mock_walker.Mock_Walker) mock_reporter := new(mock_reporter.Mock_Reporter) mock_time := new(mock_time.MockTime) - controller := new(mock_decomposer_controller.Mock_Controller) - database := new(mock_database_controller.Mock_Controller) + newWalker := func() workflow.Walker { return mockWalker } - executerObject := New(controller, database, mock_reporter, mock_time) - executionId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + executerObject := New(newWalker, playbookRepoMock, mock_reporter, mock_time) + runId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") playbookId := "playbook--d09351a2-a075-40c8-8054-0b7c423db83f" stepId := "step--81eff59f-d084-4324-9e0a-59e353dbd28f" - metadata := execution.Metadata{ExecutionId: executionId, PlaybookId: playbookId, StepId: stepId} + metadata := run.Metadata{RunId: runId, PlaybookId: playbookId, StepId: stepId} initialVariables := cacao.Variable{ Type: "string", @@ -73,25 +71,22 @@ func TestExecutePlaybook(t *testing.T) { timeNow, _ := time.Parse(layout, str) mock_time.On("Now").Return(timeNow) - database.On("GetDatabaseInstance").Return(playbookRepoMock) - controller.On("NewDecomposer").Return(mockDecomposer) - - mock_reporter.On("ReportStepStart", executionId, step, cacao.NewVariables(addedVariables), timeNow).Return() - mock_reporter.On("ReportStepEnd", executionId, step, cacao.NewVariables(returnedVariables), nil, timeNow).Return() + mock_reporter.On("ReportStepStart", metadata, step, cacao.NewVariables(addedVariables), timeNow).Return() + mock_reporter.On("ReportStepEnd", metadata, step, cacao.NewVariables(returnedVariables), nil, timeNow).Return() playbook := cacao.Playbook{ID: playbookId, PlaybookVariables: cacao.NewVariables(initialVariables)} - playbookRepoMock.On("Read", playbookId).Return(playbook, nil) - details := decomposer.ExecutionDetails{ExecutionId: executionId, + playbookRepoMock.On("Get", mock.Anything, playbookId).Return(playbook, nil) + details := workflow.Result{RunId: runId, PlaybookId: playbookId, Variables: cacao.NewVariables(returnedVariables)} playbook2 := cacao.Playbook{ID: playbookId, PlaybookVariables: cacao.NewVariables(expectedVariables)} - mockDecomposer.On("Execute", playbook2).Return(&details, nil) + mockWalker.On("Execute", playbook2).Return(&details, nil) results, err := executerObject.Execute(metadata, step, cacao.NewVariables(addedVariables)) - mockDecomposer.AssertExpectations(t) + mockWalker.AssertExpectations(t) mock_reporter.AssertExpectations(t) mock_time.AssertExpectations(t) assert.Equal(t, err, nil) diff --git a/internal/workflow/steps/playbook_action/subplaybook.go b/internal/workflow/steps/playbook_action/subplaybook.go new file mode 100644 index 00000000..08d051da --- /dev/null +++ b/internal/workflow/steps/playbook_action/subplaybook.go @@ -0,0 +1,73 @@ +package playbook_action + +import ( + "context" + "errors" + "fmt" + "reflect" + "soarca/internal/logger" + "soarca/internal/store" + "soarca/internal/workflow" + "soarca/pkg/cacao" + "soarca/internal/runs/model" + "soarca/internal/reporting/reporter" + timeUtil "soarca/pkg/utils/time" +) + +type PlaybookAction struct { + newWalker workflow.NewWalker + playbookStore storage.PlaybookStore + reporter reporter.IStepReporter + time timeUtil.ITime +} + +var component = reflect.TypeOf(PlaybookAction{}).PkgPath() +var log *logger.Log + +func init() { + log = logger.Logger(component, logger.Info, "", logger.Json) +} + +func New(newWalker workflow.NewWalker, playbookStore storage.PlaybookStore, reporter reporter.IStepReporter, time timeUtil.ITime) *PlaybookAction { + return &PlaybookAction{newWalker: newWalker, playbookStore: playbookStore, reporter: reporter, time: time} +} + +func (playbookAction *PlaybookAction) Execute(metadata run.Metadata, + step cacao.Step, + variables cacao.Variables) (cacao.Variables, error) { + log.Trace(metadata.RunId) + + playbookAction.reporter.ReportStepStart(metadata, step, variables, playbookAction.time.Now()) + + var reportVars = cacao.NewVariables() + var err error + defer func() { + playbookAction.reporter.ReportStepEnd(metadata, step, reportVars, err, playbookAction.time.Now()) + }() + + if step.Type != cacao.StepTypePlaybookAction { + err := errors.New(fmt.Sprint("step type is not of type ", cacao.StepTypePlaybookAction)) + log.Error(err) + return cacao.NewVariables(), err + } + + playbook, err := playbookAction.playbookStore.Get(context.Background(), step.PlaybookID) + if err != nil { + log.Error("failed loading the playbook from storage in playbook action") + return cacao.NewVariables(), err + } + + playbook.PlaybookVariables.Merge(variables) + + walker := playbookAction.newWalker() + result, err := walker.Execute(playbook) + if err != nil { + err = errors.New(fmt.Sprint("sub-playbook run failed with error: ", err)) + log.Error(err) + reportVars = result.Variables + return cacao.NewVariables(), err + } + reportVars = result.Variables + return result.Variables, nil + +} diff --git a/internal/workflow/walker.go b/internal/workflow/walker.go new file mode 100644 index 00000000..07d9be00 --- /dev/null +++ b/internal/workflow/walker.go @@ -0,0 +1,282 @@ +// Package workflow walks a playbook's workflow graph and runs its steps. +package workflow + +import ( + "errors" + "fmt" + "reflect" + + "soarca/internal/logger" + "soarca/internal/workflow/steps" + "soarca/pkg/cacao" + "soarca/internal/runs/model" + "soarca/internal/reporting/cases" + "soarca/internal/reporting/reporter" + "soarca/pkg/utils/guid" + timeUtil "soarca/pkg/utils/time" + + t "time" + + "github.com/google/uuid" +) + +type Empty struct{} + +var ( + component = reflect.TypeOf(Empty{}).PkgPath() + log *logger.Log +) + +// Result is the outcome of walking one playbook. +type Result struct { + RunId uuid.UUID + PlaybookId string + Variables cacao.Variables +} + +// Walker walks one playbook's workflow graph. One per run: it holds per-run state. +type Walker interface { + ExecuteAsync(playbook cacao.Playbook, results chan Result) + Execute(playbook cacao.Playbook) (*Result, error) +} + +// NewWalker builds a Walker for a single run. +type NewWalker func() Walker + +func init() { + log = logger.Logger(component, logger.Info, "", logger.Json) +} + +// New builds a walker. caseManager is optional; at most one may be supplied. +func New(action executors.IActionExecutor, + subPlaybook executors.IPlaybookExecuter, + condition executors.IConditionExecuter, + guid guid.IGuid, + reporter reporter.IWorkflowReporter, + time timeUtil.ITime, + caseManager ...cases.ICasesManager) *Walk { + + w := &Walk{ + action: action, + subPlaybook: subPlaybook, + condition: condition, + guid: guid, + reporter: reporter, + time: time, + } + if len(caseManager) > 0 { + w.caseManager = caseManager[0] + } + return w +} + +// Walk is a single playbook run in progress. +type Walk struct { + playbook cacao.Playbook + result Result + action executors.IActionExecutor + subPlaybook executors.IPlaybookExecuter + condition executors.IConditionExecuter + guid guid.IGuid + reporter reporter.IWorkflowReporter + caseManager cases.ICasesManager + time timeUtil.ITime +} + +// ExecuteAsync starts a run and publishes its identity before walking the graph. +func (w *Walk) ExecuteAsync(playbook cacao.Playbook, results chan Result) { + runId := w.guid.New() + log.Debugf("Starting run %s for Playbook %s", runId, playbook.ID) + + result := Result{runId, playbook.ID, playbook.PlaybookVariables} + w.result = result + + if results != nil { + results <- result + } + + _ = w.walk(playbook) +} + +func (w *Walk) Execute(playbook cacao.Playbook) (*Result, error) { + runId := w.guid.New() + log.Debugf("Starting run %s for Playbook %s", runId, playbook.ID) + w.result = Result{runId, playbook.ID, playbook.PlaybookVariables} + + err := w.walk(playbook) + + return &w.result, err +} + +func (w *Walk) walk(playbook cacao.Playbook) error { + + w.playbook = playbook + + stepId := playbook.WorkflowStart + + // Start case correlation and get case ID to be used in playbook + if w.caseManager != nil { + startMetadata := w.newStepMetadata(stepId) + + caseIdVar := w.caseManager.AddToExistingOrCreateNew(startMetadata, playbook) + playbook.PlaybookVariables.InsertOrReplace(caseIdVar) + log.Info("case id is set to: ", caseIdVar.Value) + } + + variables := cacao.NewVariables() + variables.Merge(playbook.PlaybookVariables) + + // Reporting workflow instantiation + w.reporter.ReportWorkflowStart(w.result.RunId, playbook, w.time.Now()) + + outputVariables, err := w.ExecuteBranch(stepId, variables) + + w.result.Variables = outputVariables + // Reporting workflow end + w.reporter.ReportWorkflowEnd(w.result.RunId, playbook, err, w.time.Now()) + + return err +} + +// ExecuteBranch walks a branch of the workflow. +// +// Runs until it finds an End step or returns an error in case there are no valid next step. +func (w *Walk) ExecuteBranch(stepId string, scopeVariables cacao.Variables) (cacao.Variables, error) { + playbook := w.playbook + log.Debug("Running branch starting from ", stepId) + + returnVariables := cacao.NewVariables() + + for { + currentStep, ok := playbook.Workflow[stepId] + if !ok { + return cacao.NewVariables(), fmt.Errorf("step with id %s not found", stepId) + } + + log.Debug("Executing step ", stepId) + + if currentStep.Type == "end" { + break + } + + // Note: likely (but not certainly) on_success and on_faliure will be reworked + // to become workflow branching properties, with the addition of a success_condition + // boolean evaluation at step level. + // Effectively, we should thus only check for existance of on_completion, and + // report run errors as such, not as playbook step failures - which will be handled + // with upcoming said on_success, on_failure, and success_condition properties + onCompletionStepId := currentStep.OnCompletion + if onCompletionStepId == "" { + onCompletionStepId = currentStep.OnSuccess + } + if onCompletionStepId == "" { + onCompletionStepId = currentStep.OnFailure + } + if _, ok := playbook.Workflow[onCompletionStepId]; !ok { + return cacao.NewVariables(), errors.New("empty completion step") + } + + outputVariables, err := w.ExecuteStep(currentStep, scopeVariables) + + if err == nil { + stepId = onCompletionStepId + returnVariables.Merge(outputVariables) + scopeVariables.Merge(outputVariables) + } else { + return cacao.NewVariables(), fmt.Errorf("playbook run failed at step [ %s ]. See step log for error information", stepId) + } + } + + return returnVariables, nil +} + +// newStepMetadata builds the run.Metadata for one invocation of stepId, +// minting a fresh StepRunId each time it is called. Call it once per +// actual dispatch of a step (including once per while-loop iteration), never +// reuse a previously-built value across separate invocations. +func (w *Walk) newStepMetadata(stepId string) run.Metadata { + return run.Metadata{ + RunId: w.result.RunId, + PlaybookId: w.result.PlaybookId, + StepId: stepId, + StepRunId: w.guid.New(), + } +} + +// ExecuteStep runs a single step within the workflow. +func (w *Walk) ExecuteStep(step cacao.Step, scopeVariables cacao.Variables) (cacao.Variables, error) { + log.Debug("Running step type ", step.Type) + + log.Trace("Delay is set to: ", step.Delay) + w.time.Sleep(t.Duration(step.Delay) * t.Millisecond) + + // Combine parent scope and Step variables + variables := cacao.NewVariables() + variables.Merge(scopeVariables) + variables.Merge(step.StepVariables) + + switch step.Type { + case cacao.StepTypeAction: + metadata := w.newStepMetadata(step.ID) + actionMetadata := executors.PlaybookStepMetadata{ + Step: step, + Targets: w.playbook.TargetDefinitions, + Auth: w.playbook.AuthenticationInfoDefinitions, + Agent: w.playbook.AgentDefinitions[step.Agent], + Variables: variables, + } + return w.action.Execute(metadata, actionMetadata) + case cacao.StepTypePlaybookAction: + metadata := w.newStepMetadata(step.ID) + return w.subPlaybook.Execute(metadata, step, variables) + case cacao.StepTypeIfCondition: + return w.executeIfCondition(step, variables) + case cacao.StepTypeWhileCondition: + return w.executeLoop(step, variables) + default: + // NOTE: This currently silently handles unknown step types. Should we return an error instead? + return cacao.NewVariables(), nil //errors.ErrUnsupported + } +} + +func (w *Walk) executeIfCondition(step cacao.Step, + variables cacao.Variables) (cacao.Variables, error) { + metadata := w.newStepMetadata(step.ID) + stepId, branch, err := w.condition.Execute(metadata, + executors.Context{Step: step, Variables: variables}) + if err != nil { + return cacao.NewVariables(), err + } + if branch { + return w.ExecuteBranch(stepId, variables) + } + return variables, nil +} + +func (w *Walk) executeLoop(step cacao.Step, + variables cacao.Variables) (cacao.Variables, error) { + + loop := true + + for loop { + // A fresh StepRunId per iteration: each pass through the loop + // re-evaluates this same while-condition step. + metadata := w.newStepMetadata(step.ID) + stepId, branch, err := w.condition.Execute(metadata, + executors.Context{Step: step, Variables: variables}) + if err != nil { + return cacao.NewVariables(), err + } + loop = branch + + if loop { + branchVariables, err := w.ExecuteBranch(stepId, variables) + if err != nil { + return variables, err + } + variables.Merge(branchVariables) + } + + } + return variables, nil +} diff --git a/pkg/core/decomposer/decomposer_test.go b/internal/workflow/walker_test.go similarity index 88% rename from pkg/core/decomposer/decomposer_test.go rename to internal/workflow/walker_test.go index ac0923ab..7dfe3043 100644 --- a/pkg/core/decomposer/decomposer_test.go +++ b/internal/workflow/walker_test.go @@ -1,4 +1,4 @@ -package decomposer +package workflow import ( "errors" @@ -6,9 +6,9 @@ import ( "testing" "time" - "soarca/pkg/core/executors" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" + "soarca/internal/workflow/steps" + "soarca/pkg/cacao" + "soarca/internal/runs/model" "soarca/test/unittest/mocks/mock_executor" mock_condition_executor "soarca/test/unittest/mocks/mock_executor/condition" mock_playbook_action_executor "soarca/test/unittest/mocks/mock_executor/playbook_action" @@ -93,10 +93,10 @@ func TestExecutePlaybook(t *testing.T) { Workflow: map[string]cacao.Step{step1.ID: step1, end.ID: end}, } - executionId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") - metaStep1 := execution.Metadata{ExecutionId: executionId, PlaybookId: "test", StepId: step1.ID} + runId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + metaStep1 := run.Metadata{RunId: runId, PlaybookId: "test", StepId: step1.ID, StepRunId: runId} - uuid_mock.On("New").Return(executionId) + uuid_mock.On("New").Return(runId) playbookStepMetadata := executors.PlaybookStepMetadata{ Step: step1, @@ -111,16 +111,16 @@ func TestExecutePlaybook(t *testing.T) { timeNow, _ := time.Parse(layout, str) mock_time.On("Now").Return(timeNow) - mock_reporter.On("ReportWorkflowStart", executionId, playbook, timeNow).Return() + mock_reporter.On("ReportWorkflowStart", runId, playbook, timeNow).Return() mock_time.On("Sleep", time.Millisecond*10).Return() - mock_reporter.On("ReportWorkflowEnd", executionId, playbook, nil, timeNow).Return() + mock_reporter.On("ReportWorkflowEnd", runId, playbook, nil, timeNow).Return() mock_action_executor.On("Execute", metaStep1, playbookStepMetadata).Return(cacao.NewVariables(cacao.Variable{Name: "return", Value: "value"}), nil) details, err := decomposer.Execute(playbook) uuid_mock.AssertExpectations(t) fmt.Println(err) assert.Equal(t, err, nil) - assert.Equal(t, details.ExecutionId, executionId) + assert.Equal(t, details.RunId, runId) mock_action_executor.AssertExpectations(t) mock_reporter.AssertExpectations(t) mock_time.AssertExpectations(t) @@ -233,11 +233,11 @@ func TestExecutePlaybookMultiStep(t *testing.T) { Workflow: map[string]cacao.Step{step1.ID: step1, step2.ID: step2, step3.ID: step3, end.ID: end}, } - executionId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") - metaStep1 := execution.Metadata{ExecutionId: executionId, PlaybookId: "test", StepId: step1.ID} - metaStep2 := execution.Metadata{ExecutionId: executionId, PlaybookId: "test", StepId: step2.ID} + runId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + metaStep1 := run.Metadata{RunId: runId, PlaybookId: "test", StepId: step1.ID, StepRunId: runId} + metaStep2 := run.Metadata{RunId: runId, PlaybookId: "test", StepId: step2.ID, StepRunId: runId} - uuid_mock.On("New").Return(executionId) + uuid_mock.On("New").Return(runId) firstResult := cacao.Variable{Name: "result", Value: "value"} @@ -254,9 +254,9 @@ func TestExecutePlaybookMultiStep(t *testing.T) { timeNow, _ := time.Parse(layout, str) mock_time.On("Now").Return(timeNow) - mock_reporter.On("ReportWorkflowStart", executionId, playbook, timeNow).Return() + mock_reporter.On("ReportWorkflowStart", runId, playbook, timeNow).Return() mock_time.On("Sleep", time.Millisecond*0).Return() - mock_reporter.On("ReportWorkflowEnd", executionId, playbook, nil, timeNow).Return() + mock_reporter.On("ReportWorkflowEnd", runId, playbook, nil, timeNow).Return() mock_action_executor.On("Execute", metaStep1, playbookStepMetadata1).Return(cacao.NewVariables(firstResult), nil) playbookStepMetadata2 := executors.PlaybookStepMetadata{ @@ -273,7 +273,7 @@ func TestExecutePlaybookMultiStep(t *testing.T) { uuid_mock.AssertExpectations(t) fmt.Println(err) assert.Equal(t, err, nil) - assert.Equal(t, details.ExecutionId, executionId) + assert.Equal(t, details.RunId, runId) mock_action_executor.AssertExpectations(t) mock_reporter.AssertExpectations(t) @@ -358,14 +358,14 @@ func TestExecuteEmptyMultiStep(t *testing.T) { uuid_mock2.AssertExpectations(t) fmt.Println(err) assert.Equal(t, err, errors.New("empty completion step")) - assert.Equal(t, returnedId.ExecutionId, id) + assert.Equal(t, returnedId.RunId, id) mock_action_executor2.AssertExpectations(t) mock_reporter.AssertExpectations(t) } /* -An error-raising step execution will raise an error for the playbook execution, -Thus reported as execution failure. +An error-raising step run will raise an error for the playbook run, +Thus reported as run failure. */ func TestFailingStepResultsInFailingPlaybook(t *testing.T) { mock_action_executor := new(mock_executor.Mock_Action_Executor) @@ -483,12 +483,12 @@ func TestFailingStepResultsInFailingPlaybook(t *testing.T) { Workflow: map[string]cacao.Step{step0.ID: step0, step1.ID: step1, step2.ID: step2, step3.ID: step3, end.ID: end}, } - executionId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") - metaStep1 := execution.Metadata{ExecutionId: executionId, PlaybookId: "test", StepId: step1.ID} - metaStep2 := execution.Metadata{ExecutionId: executionId, PlaybookId: "test", StepId: step2.ID} - metaStep3 := execution.Metadata{ExecutionId: executionId, PlaybookId: "test", StepId: step3.ID} + runId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + metaStep1 := run.Metadata{RunId: runId, PlaybookId: "test", StepId: step1.ID, StepRunId: runId} + metaStep2 := run.Metadata{RunId: runId, PlaybookId: "test", StepId: step2.ID, StepRunId: runId} + metaStep3 := run.Metadata{RunId: runId, PlaybookId: "test", StepId: step3.ID, StepRunId: runId} - uuid_mock.On("New").Return(executionId) + uuid_mock.On("New").Return(runId) firstResult := cacao.Variable{Name: "result", Value: "value"} @@ -505,7 +505,7 @@ func TestFailingStepResultsInFailingPlaybook(t *testing.T) { timeNow, _ := time.Parse(layout, str) mock_time.On("Now").Return(timeNow) - mock_reporter.On("ReportWorkflowStart", executionId, playbook, timeNow).Return() + mock_reporter.On("ReportWorkflowStart", runId, playbook, timeNow).Return() mock_time.On("Sleep", time.Millisecond*0).Return() mock_action_executor.On("Execute", metaStep1, playbookStepMetadata1).Return(cacao.NewVariables(firstResult), nil) @@ -529,15 +529,15 @@ func TestFailingStepResultsInFailingPlaybook(t *testing.T) { mock_action_executor.On("Execute", metaStep3, playbookStepMetadata3).Return(cacao.NewVariables(), errors.New("everything broke")) mock_time.On("Now").Return(timeNow) - expectedError := errors.New("playbook execution failed at step [ action--test3 ]. See step log for error information") - mock_reporter.On("ReportWorkflowEnd", executionId, playbook, expectedError, timeNow).Return() + expectedError := errors.New("playbook run failed at step [ action--test3 ]. See step log for error information") + mock_reporter.On("ReportWorkflowEnd", runId, playbook, expectedError, timeNow).Return() _, err := decomposer.Execute(playbook) t.Log(err) uuid_mock.AssertExpectations(t) assert.Equal(t, err, expectedError) // Confirms that the expectedError has been raised and reported correctly. - // If the Execution had not actually raised the expected error, the + // If the Run had not actually raised the expected error, the // mock_reporter.On("ReportWorkflowEnd", ..., expectedError), would not match mock_action_executor.AssertExpectations(t) mock_reporter.AssertExpectations(t) @@ -607,7 +607,7 @@ func TestExecuteIllegalMultiStep(t *testing.T) { mock_reporter.AssertExpectations(t) fmt.Println(err) assert.Equal(t, err, errors.New("empty completion step")) - assert.Equal(t, returnedId.ExecutionId, id) + assert.Equal(t, returnedId.RunId, id) mock_action_executor2.AssertExpectations(t) } @@ -654,18 +654,18 @@ func TestExecutePlaybookAction(t *testing.T) { Workflow: map[string]cacao.Step{step1.ID: step1, end.ID: end}, } - executionId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") - metaStep1 := execution.Metadata{ExecutionId: executionId, PlaybookId: "test", StepId: step1.ID} + runId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + metaStep1 := run.Metadata{RunId: runId, PlaybookId: "test", StepId: step1.ID, StepRunId: runId} layout := "2006-01-02T15:04:05.000Z" str := "2014-11-12T11:45:26.371Z" timeNow, _ := time.Parse(layout, str) mock_time.On("Now").Return(timeNow) - uuid_mock.On("New").Return(executionId) - mock_reporter.On("ReportWorkflowStart", executionId, playbook, timeNow).Return() + uuid_mock.On("New").Return(runId) + mock_reporter.On("ReportWorkflowStart", runId, playbook, timeNow).Return() mock_time.On("Sleep", time.Millisecond*0).Return() - mock_reporter.On("ReportWorkflowEnd", executionId, playbook, nil, timeNow).Return() + mock_reporter.On("ReportWorkflowEnd", runId, playbook, nil, timeNow).Return() mock_playbook_action_executor.On("Execute", metaStep1, @@ -676,7 +676,7 @@ func TestExecutePlaybookAction(t *testing.T) { uuid_mock.AssertExpectations(t) fmt.Println(err) assert.Equal(t, err, nil) - assert.Equal(t, details.ExecutionId, executionId) + assert.Equal(t, details.RunId, runId) mock_reporter.AssertExpectations(t) mock_action_executor.AssertExpectations(t) value, found := details.Variables.Find("return") @@ -815,11 +815,11 @@ func TestExecuteIfCondition(t *testing.T) { timeNow, _ := time.Parse(layout, str) mock_time.On("Now").Return(timeNow) - executionId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") - metaStepIf := execution.Metadata{ExecutionId: executionId, PlaybookId: "test", StepId: stepIf.ID} + runId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + metaStepIf := run.Metadata{RunId: runId, PlaybookId: "test", StepId: stepIf.ID, StepRunId: runId} - uuid_mock.On("New").Return(executionId) - mock_reporter.On("ReportWorkflowStart", executionId, playbook, timeNow).Return() + uuid_mock.On("New").Return(runId) + mock_reporter.On("ReportWorkflowStart", runId, playbook, timeNow).Return() mock_time.On("Sleep", time.Millisecond*0).Return() mock_condition_executor.On("Execute", @@ -836,7 +836,7 @@ func TestExecuteIfCondition(t *testing.T) { Variables: cacao.NewVariables(expectedVariables), } - metaStepTrue := execution.Metadata{ExecutionId: executionId, PlaybookId: "test", StepId: stepTrue.ID} + metaStepTrue := run.Metadata{RunId: runId, PlaybookId: "test", StepId: stepTrue.ID, StepRunId: runId} mock_time.On("Sleep", time.Millisecond*0).Return() mock_action_executor.On("Execute", @@ -851,25 +851,25 @@ func TestExecuteIfCondition(t *testing.T) { Variables: cacao.NewVariables(expectedVariables, expectedVariables2), } - metaStepCompletion := execution.Metadata{ExecutionId: executionId, PlaybookId: "test", StepId: stepCompletion.ID} + metaStepCompletion := run.Metadata{RunId: runId, PlaybookId: "test", StepId: stepCompletion.ID, StepRunId: runId} mock_time.On("Sleep", time.Millisecond*0).Return() mock_action_executor.On("Execute", metaStepCompletion, stepCompletionDetails).Return(cacao.NewVariables(), nil) - mock_reporter.On("ReportWorkflowEnd", executionId, playbook, nil, timeNow).Return() + mock_reporter.On("ReportWorkflowEnd", runId, playbook, nil, timeNow).Return() details, err := decomposer.Execute(playbook) uuid_mock.AssertExpectations(t) fmt.Println(err) assert.Equal(t, err, nil) - assert.Equal(t, details.ExecutionId, executionId) + assert.Equal(t, details.RunId, runId) mock_reporter.AssertExpectations(t) mock_condition_executor.AssertExpectations(t) mock_action_executor.AssertExpectations(t) } -func TestDelayStepExecution(t *testing.T) { +func TestDelayStepRun(t *testing.T) { mock_action_executor := new(mock_executor.Mock_Action_Executor) mock_playbook_action_executor := new(mock_playbook_action_executor.Mock_PlaybookActionExecutor) mock_condition_executor := new(mock_condition_executor.Mock_Condition) @@ -908,8 +908,9 @@ func TestDelayStepExecution(t *testing.T) { mock_reporter, mock_time) - executionId, _ := uuid.Parse("00000000-0000-0000-0000-000000000000") - metaStep1 := execution.Metadata{ExecutionId: executionId, PlaybookId: "", StepId: step1.ID} + runId, _ := uuid.Parse("00000000-0000-0000-0000-000000000000") + metaStep1 := run.Metadata{RunId: runId, PlaybookId: "", StepId: step1.ID, StepRunId: runId} + uuid_mock.On("New").Return(runId) playbookStepMetadata := executors.PlaybookStepMetadata{ Step: step1, Variables: cacao.NewVariables(expectedVariables), @@ -923,7 +924,7 @@ func TestDelayStepExecution(t *testing.T) { } -func TestDelayStepNegativeTimeExecution(t *testing.T) { +func TestDelayStepNegativeTimeRun(t *testing.T) { mock_action_executor := new(mock_executor.Mock_Action_Executor) mock_playbook_action_executor := new(mock_playbook_action_executor.Mock_PlaybookActionExecutor) mock_condition_executor := new(mock_condition_executor.Mock_Condition) @@ -962,8 +963,9 @@ func TestDelayStepNegativeTimeExecution(t *testing.T) { mock_reporter, mock_time) - executionId, _ := uuid.Parse("00000000-0000-0000-0000-000000000000") - metaStep1 := execution.Metadata{ExecutionId: executionId, PlaybookId: "", StepId: step1.ID} + runId, _ := uuid.Parse("00000000-0000-0000-0000-000000000000") + metaStep1 := run.Metadata{RunId: runId, PlaybookId: "", StepId: step1.ID, StepRunId: runId} + uuid_mock.On("New").Return(runId) playbookStepMetadata := executors.PlaybookStepMetadata{ Step: step1, Variables: cacao.NewVariables(expectedVariables), @@ -1089,11 +1091,11 @@ func TestExecuteWhileCondition(t *testing.T) { timeNow, _ := time.Parse(layout, str) mock_time.On("Now").Return(timeNow) - executionId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") - metaStepIf := execution.Metadata{ExecutionId: executionId, PlaybookId: "test", StepId: stepWhile.ID} + runId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + metaStepIf := run.Metadata{RunId: runId, PlaybookId: "test", StepId: stepWhile.ID, StepRunId: runId} - uuid_mock.On("New").Return(executionId) - mock_reporter.On("ReportWorkflowStart", executionId, playbook, timeNow).Return() + uuid_mock.On("New").Return(runId) + mock_reporter.On("ReportWorkflowStart", runId, playbook, timeNow).Return() mock_time.On("Sleep", time.Millisecond*0).Return() mock_condition_executor.On("Execute", @@ -1110,7 +1112,7 @@ func TestExecuteWhileCondition(t *testing.T) { Variables: cacao.NewVariables(expectedVariables), } - metaStepTrue := execution.Metadata{ExecutionId: executionId, PlaybookId: "test", StepId: stepTrue.ID} + metaStepTrue := run.Metadata{RunId: runId, PlaybookId: "test", StepId: stepTrue.ID, StepRunId: runId} mock_time.On("Sleep", time.Millisecond*0).Return() mock_action_executor.On("Execute", @@ -1131,18 +1133,18 @@ func TestExecuteWhileCondition(t *testing.T) { Variables: cacao.NewVariables(expectedVariables, expectedVariables2), } - metaStepCompletion := execution.Metadata{ExecutionId: executionId, PlaybookId: "test", StepId: stepCompletion.ID} + metaStepCompletion := run.Metadata{RunId: runId, PlaybookId: "test", StepId: stepCompletion.ID, StepRunId: runId} mock_time.On("Sleep", time.Millisecond*0).Return() mock_action_executor.On("Execute", metaStepCompletion, stepCompletionDetails).Return(cacao.NewVariables(), nil) - mock_reporter.On("ReportWorkflowEnd", executionId, playbook, nil, timeNow).Return() + mock_reporter.On("ReportWorkflowEnd", runId, playbook, nil, timeNow).Return() details, err := decomposer.Execute(playbook) uuid_mock.AssertExpectations(t) fmt.Println(err) assert.Equal(t, err, nil) - assert.Equal(t, details.ExecutionId, executionId) + assert.Equal(t, details.RunId, runId) mock_reporter.AssertExpectations(t) mock_condition_executor.AssertExpectations(t) mock_action_executor.AssertExpectations(t) diff --git a/makefile b/makefile index c2d7abe8..d7372cff 100644 --- a/makefile +++ b/makefile @@ -1,4 +1,4 @@ -.PHONY: all test integration-test ci-test clean build docker run pre-docker-build swagger sbom +.PHONY: all test integration-test manual-test ci-test clean build docker run pre-docker-build swagger sbom BINARY_NAME=soarca DIRECTORY = $(sort $(dir $(wildcard ./test/*/))) @@ -28,12 +28,17 @@ lint: swagger build: swagger CGO_ENABLED=0 go build -o ./build/soarca $(GOFLAGS) ./cmd/soarca/main.go +# Unit tests: no external services required, safe to run anywhere. test: swagger - go test ./pkg/... -v - go test ./internal/... -v + go test ./... -v +# Requires the test services in deployments/docker/testing (httpbin, ssh, thehive). integration-test: swagger - go test ./test/integration/... -v + go test -tags=integration ./... -v + +# Requires a special environment (Windows/PowerShell host, live TheHive). +manual-test: swagger + go test -tags=manual ./... -v ci-test: test integration-test diff --git a/pkg/api/api.go b/pkg/api/api.go deleted file mode 100644 index f1dc9698..00000000 --- a/pkg/api/api.go +++ /dev/null @@ -1,146 +0,0 @@ -package api - -import ( - "reflect" - open_api "soarca/api" - "soarca/internal/controller/database" - "soarca/internal/controller/decomposer_controller" - "soarca/internal/controller/informer" - "soarca/internal/logger" - playbook_handler "soarca/pkg/api/playbook" - reporter_handler "soarca/pkg/api/reporter" - status_handler "soarca/pkg/api/status" - "soarca/pkg/core/capability/manual/interaction" - - manual_handler "soarca/pkg/api/manual" - - trigger_handler "soarca/pkg/api/trigger" - - "github.com/gin-contrib/cors" - gin "github.com/gin-gonic/gin" - swaggerfiles "github.com/swaggo/files" - ginSwagger "github.com/swaggo/gin-swagger" -) - -var log *logger.Log - -type Empty struct{} - -func init() { - log = logger.Logger(reflect.TypeOf(Empty{}).PkgPath(), logger.Info, "", logger.Json) -} - -func Database(app *gin.Engine, - controller database.IController, -) error { - log.Trace("Setting up playbook routes") - PlaybookRoutes(app, controller) - return nil -} - -func Logging(app *gin.Engine) { - // app.Use(middelware.LoggingMiddleware(log.Logger)) -} - -func Reporter(app *gin.Engine, informer informer.IExecutionInformer) error { - log.Trace("Setting up reporter routes") - ReporterRoutes(app, informer) - return nil -} - -func Manual(app *gin.Engine, interaction interaction.IInteractionStorage) { - log.Trace("Setting up manual routes") - manualHandler := manual_handler.NewManualHandler(interaction) - ManualRoutes(app, manualHandler) -} - -func Api(app *gin.Engine, - controller decomposer_controller.IController, - database database.IController, -) error { - log.Trace("Trying to setup all Routes") - // gin.SetMode(gin.ReleaseMode) - triggerHandler := trigger_handler.NewTriggerHandler(controller, database) - TriggerRoutes(app, triggerHandler) - StatusRoutes(app) - - return nil -} - -func Cors(app *gin.Engine, origins []string) { - config := cors.DefaultConfig() - config.AllowOrigins = origins - app.Use(cors.New(config)) -} - -func Swagger(app *gin.Engine) { - swaggerRoutes(app) -} - -func swaggerRoutes(route *gin.Engine) { - open_api.SwaggerInfo.BasePath = "/" - swaggerRoutes := route.Group("/swagger") - { - swaggerRoutes.GET("/*any", ginSwagger.WrapHandler(swaggerfiles.Handler)) - } -} - -// Main Router for the following endpoints: -// GET /playbook -// POST /playbook -// GET /playbook/playbook-id -// PUT /playbook/playbook-id -// DELETE /playbook/playbook-id -func PlaybookRoutes(route *gin.Engine, controller database.IController) { - playbookHandler := playbook_handler.NewPlaybookHandler(controller) - playbookRoutes := route.Group("/playbook") - { - playbookRoutes.GET("/", playbookHandler.GetAllPlaybooks) - playbookRoutes.POST("/", playbookHandler.SubmitPlaybook) - playbookRoutes.GET("/meta/", playbookHandler.GetAllPlaybookMetas) - playbookRoutes.GET("/:id", playbookHandler.GetPlaybookByID) - playbookRoutes.PUT("/:id", playbookHandler.UpdatePlaybookByID) - playbookRoutes.DELETE("/:id", playbookHandler.DeleteByPlaybookID) - - } -} - -// Main Router for the following endpoints: -// GET /reporter -// GET /reporter/{execution-id} -func ReporterRoutes(route *gin.Engine, informer informer.IExecutionInformer) { - reportHandler := reporter_handler.NewReportHandler(informer) - reportRoutes := route.Group("/reporter") - { - reportRoutes.GET("/", reportHandler.GetExecutions) - reportRoutes.GET("/:id", reportHandler.GetExecutionReport) - } -} - -// GET /status -// GET /status/ping -func StatusRoutes(route *gin.Engine) { - router := route.Group("/status") - { - router.GET("/", status_handler.GetApi) - router.GET("/ping", status_handler.GetPong) - - } -} - -func TriggerRoutes(route *gin.Engine, triggerHandler *trigger_handler.TriggerHandler) { - triggerRoutes := route.Group("/trigger") - { - triggerRoutes.POST("/playbook", triggerHandler.Execute) - triggerRoutes.POST("/playbook/:id", triggerHandler.ExecuteById) - } -} - -func ManualRoutes(route *gin.Engine, manualHandler *manual_handler.ManualHandler) { - manualRoutes := route.Group("/manual") - { - manualRoutes.GET("/", manualHandler.GetPendingCommands) - manualRoutes.GET(":exec_id/:step_id", manualHandler.GetPendingCommand) - manualRoutes.POST("/continue", manualHandler.PostContinue) - } -} diff --git a/pkg/api/manual/manual_api.go b/pkg/api/manual/manual_api.go deleted file mode 100644 index ff4bb918..00000000 --- a/pkg/api/manual/manual_api.go +++ /dev/null @@ -1,271 +0,0 @@ -package manual - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "reflect" - "soarca/internal/logger" - "soarca/pkg/core/capability/manual/interaction" - "soarca/pkg/models/api" - "soarca/pkg/models/execution" - "soarca/pkg/models/manual" - - "github.com/gin-gonic/gin" - "github.com/google/uuid" - - apiError "soarca/pkg/api/error" -) - -// Notes: -// A manual command in CACAO is simply the operation: -// { post_message; wait_for_response (returning a result) } -// The manual API expose general manual executions wide information -// Thus, we need a ManualHandler that uses an IInteractionStorage, implemented by interactionCapability -// The API routes will invoke the ManualHandler.interactionCapability interface instance -// The InteractionCapability manages the manual command infromation and status, like a cache. And interfaces any interactor type (e.g. API, integration) - -// It is always either only the internal API, or the internal API and ONE integration for manual. -// Env variable: can only have one active manual interactor. -// -// In light of this, for hierarchical and distributed playbooks executions (via multiple playbook actions), -// there will be ONE manual integration (besides internal API) per every ONE SOARCA instance. - -var log *logger.Log - -type Empty struct{} - -func init() { - log = logger.Logger(reflect.TypeOf(Empty{}).PkgPath(), logger.Info, "", logger.Json) -} - -type ManualHandler struct { - interactionCapability interaction.IInteractionStorage -} - -func NewManualHandler(interaction interaction.IInteractionStorage) *ManualHandler { - return &ManualHandler{interactionCapability: interaction} -} - -// manual -// -// @Summary get all pending manual commands that still needs values to be returned -// @Schemes -// @Description get all pending manual commands that still needs values to be returned -// @Tags manual -// @Accept json -// @Produce json -// @Success 200 {object} []api.InteractionCommandData -// @failure 400 {object} []api.InteractionCommandData -// @Router /manual/ [GET] -func (manualHandler *ManualHandler) GetPendingCommands(g *gin.Context) { - commands, err := manualHandler.interactionCapability.GetPendingCommands() - if err != nil { - log.Error(err) - apiError.SendErrorResponse(g, http.StatusInternalServerError, - "Failed get pending manual commands", - "GET /manual/", "") - return - } - - response := []api.InteractionCommandData{} - for _, command := range commands { - response = append(response, manualHandler.parseCommandInfoToResponse(command)) - } - - g.JSON(http.StatusOK, - response) -} - -// manual -// -// @Summary get a specific manual command that still needs a value to be returned -// @Schemes -// @Description get a specific manual command that still needs a value to be returned -// @Tags manual -// @Accept json -// @Produce json -// @Param exec_id path string true "execution ID" -// @Param step_id path string true "step ID" -// @Success 200 {object} api.InteractionCommandData -// @failure 400 {object} api.Error -// @Router /manual/{exec_id}/{step_id} [GET] -func (manualHandler *ManualHandler) GetPendingCommand(g *gin.Context) { - execution_id := g.Param("exec_id") - step_id := g.Param("step_id") - execId, err := uuid.Parse(execution_id) - if err != nil { - log.Error(err) - apiError.SendErrorResponse(g, http.StatusBadRequest, - "Failed to parse execution ID", - "GET /manual/"+execution_id+"/"+step_id, "") - return - } - - executionMetadata := execution.Metadata{ExecutionId: execId, StepId: step_id} - commandData, err := manualHandler.interactionCapability.GetPendingCommand(executionMetadata) - if err != nil { - log.Error(err) - code := http.StatusBadRequest - if errors.Is(err, manual.ErrorPendingCommandNotFound{}) { - code = http.StatusNotFound - } - apiError.SendErrorResponse(g, code, - "Failed to provide pending manual command", - "GET /manual/"+execution_id+"/"+step_id, "") - return - } - - commandInfo := manualHandler.parseCommandInfoToResponse(commandData) - - g.JSON(http.StatusOK, commandInfo) -} - -// manual -// -// @Summary updates the value of a variable according to the manual interaction -// @Schemes -// @Description updates the value of a variable according to the manual interaction -// @Tags manual -// @Accept json -// @Produce json -// @Param exec_id path string true "execution ID" -// @Param step_id path string true "step ID" -// @Param data body api.ManualOutArgsUpdatePayload true "playbook" -// @Success 200 {object} api.Execution -// @failure 400 {object} api.Error -// @Router /manual/continue [POST] -func (manualHandler *ManualHandler) PostContinue(g *gin.Context) { - - byteData, err := io.ReadAll(g.Request.Body) - if err != nil { - log.Error("failed") - apiError.SendErrorResponse(g, http.StatusBadRequest, - "Failed to read json", - "POST /manual/continue", "") - return - } - - outArgsUpdate, err := manualHandler.parseManualOutArgsUpdate(byteData) - if err != nil { - apiError.SendErrorResponse(g, http.StatusBadRequest, - fmt.Sprint(fmt.Errorf("failed to parse manual out args payload: %w", err)), - "POST /manual/continue", err.Error()) - return - } - - interactionResponse, err := manualHandler.parseManualOutArgsToInteractionResponse(outArgsUpdate) - if err != nil { - apiError.SendErrorResponse(g, http.StatusBadRequest, - "Failed to parse response", - "POST /manual/continue", err.Error()) - return - } - - err = manualHandler.interactionCapability.PostContinue(interactionResponse) - if err != nil { - log.Error(err) - code := http.StatusBadRequest - msg := "Failed to post the continue request" - if errors.Is(err, manual.ErrorPendingCommandNotFound{}) { - code = http.StatusNotFound - msg = "Pending command not found" - } else if errors.Is(err, manual.ErrorNonMatchingOutArgs{}) { - code = http.StatusBadRequest - msg = "Provided out args don't match with expected" - } - apiError.SendErrorResponse(g, code, - msg, - "POST /manual/continue", "") - return - } - executionId, err := uuid.Parse(outArgsUpdate.ExecutionId) - if err != nil { - apiError.SendErrorResponse(g, http.StatusInternalServerError, - "Failed to parse execution ID", - "POST /manual/continue", "") - return - } - - g.JSON( - http.StatusOK, - api.Execution{ - ExecutionId: executionId, - PlaybookId: outArgsUpdate.PlaybookId, - }) -} - -// ############################################################################ -// Utility -// ############################################################################ - -func (manualHandler *ManualHandler) parseManualOutArgsUpdate(postData []byte) (api.ManualOutArgsUpdatePayload, error) { - decoder := json.NewDecoder(bytes.NewReader(postData)) - decoder.DisallowUnknownFields() - var outArgsUpdate api.ManualOutArgsUpdatePayload - err := decoder.Decode(&outArgsUpdate) - if err != nil { - errorString := fmt.Errorf("failed to unmarshal JSON: %w", err) - log.Error(errorString) - return api.ManualOutArgsUpdatePayload{}, errorString - } - - // Check if variable names match - for varName, variable := range outArgsUpdate.ResponseOutArgs { - if varName != variable.Name { - errorString := fmt.Errorf( - "variable name mismatch for variable %s: has different name property: %s", - varName, variable.Name) - log.Error(errorString) - return api.ManualOutArgsUpdatePayload{}, errorString - } - } - - return outArgsUpdate, nil -} - -func (manualHandler *ManualHandler) parseCommandInfoToResponse(commandInfo manual.CommandInfo) api.InteractionCommandData { - commandText := commandInfo.Context.Command.Command - isBase64 := false - if len(commandInfo.Context.Command.CommandB64) > 0 { - commandText = commandInfo.Context.Command.CommandB64 - isBase64 = true - } - - response := api.InteractionCommandData{ - Type: "manual-command-info", - ExecutionId: commandInfo.Metadata.ExecutionId.String(), - PlaybookId: commandInfo.Metadata.PlaybookId, - StepId: commandInfo.Metadata.StepId, - Description: commandInfo.Context.Command.Description, - Command: commandText, - CommandIsBase64: isBase64, - Target: commandInfo.Context.Target, - OutVariables: commandInfo.OutArgsVariables, - } - - return response -} - -func (manualHandler *ManualHandler) parseManualOutArgsToInteractionResponse(response api.ManualOutArgsUpdatePayload) (manual.InteractionResponse, error) { - executionId, err := uuid.Parse(response.ExecutionId) - if err != nil { - return manual.InteractionResponse{}, err - } - - interactionResponse := manual.InteractionResponse{ - Metadata: execution.Metadata{ - ExecutionId: executionId, - PlaybookId: response.PlaybookId, - StepId: response.StepId, - }, - ResponseStatus: response.ResponseStatus, - OutArgsVariables: response.ResponseOutArgs, - ResponseError: nil, - } - - return interactionResponse, nil -} diff --git a/pkg/api/manual/manual_api_utils_test.go b/pkg/api/manual/manual_api_utils_test.go deleted file mode 100644 index 9b148572..00000000 --- a/pkg/api/manual/manual_api_utils_test.go +++ /dev/null @@ -1,194 +0,0 @@ -package manual - -import ( - "errors" - "reflect" - "soarca/pkg/core/capability" - "soarca/pkg/models/api" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" - "soarca/pkg/models/manual" - "soarca/test/unittest/mocks/mock_interaction_storage" - "testing" - - "github.com/go-playground/assert/v2" - "github.com/google/uuid" -) - -func TestParseManualOutArgsUpdate(t *testing.T) { - manualHandler := NewManualHandler(&mock_interaction_storage.MockInteractionStorage{}) - - testExecId := "50b6d52c-6efc-4516-a242-dfbc5c89d421" - testStepId := "61a4d52c-6efc-4516-a242-dfbc5c89d312" - testPlaybookId := "21a4d52c-6efc-4516-a242-dfbc5c89d312" - - jsonPayload := `{"type":"out-args-update","execution_id":"50b6d52c-6efc-4516-a242-dfbc5c89d421","playbook_id":"21a4d52c-6efc-4516-a242-dfbc5c89d312","step_id":"61a4d52c-6efc-4516-a242-dfbc5c89d312","response_status":"success","response_out_args":{"__test__":{"type":"string","name":"__test__","value":"updated!"}}}` - bytesPayload := []byte(jsonPayload) - - outVariable := cacao.Variable{Type: "string", Name: "__test__", Value: "updated!"} - outVariables := map[string]cacao.Variable{"__test__": outVariable} - - expectedPayload := api.ManualOutArgsUpdatePayload{ - Type: "out-args-update", - ExecutionId: testExecId, - PlaybookId: testPlaybookId, - StepId: testStepId, - ResponseStatus: manual.ManualResponseSuccessStatus, - ResponseOutArgs: outVariables, - } - - receivedPayload, err := manualHandler.parseManualOutArgsUpdate(bytesPayload) - if err != nil { - t.Fatalf("failed to parse manual out args update: %v", err) - } - assert.Equal(t, receivedPayload, expectedPayload) -} - -func TestParseManualOutArgsUpdateFailOnVariablesNames(t *testing.T) { - manualHandler := NewManualHandler(&mock_interaction_storage.MockInteractionStorage{}) - - jsonPayload := `{"type":"out-args-update","execution_id":"50b6d52c-6efc-4516-a242-dfbc5c89d421","playbook_id":"21a4d52c-6efc-4516-a242-dfbc5c89d312","step_id":"61a4d52c-6efc-4516-a242-dfbc5c89d312","response_status":"success","response_out_args":{"__test__":{"type":"string","name":"__wrong_name__","value":"updated!"}}}` - bytesPayload := []byte(jsonPayload) - - expecedErr := errors.New("variable name mismatch for variable __test__: has different name property: __wrong_name__") - _, err := manualHandler.parseManualOutArgsUpdate(bytesPayload) - if err == nil { - t.Log("an error for non-matching variables names should have been raised") - t.Fail() - } - - assert.Equal(t, err, expecedErr) -} - -func TestParseManualOutArgsUpdateFailOnInvalidModel(t *testing.T) { - manualHandler := NewManualHandler(&mock_interaction_storage.MockInteractionStorage{}) - - jsonPayload := `{"invalidProperty":"out-args-update","execution_id":"50b6d52c-6efc-4516-a242-dfbc5c89d421","playbook_id":"21a4d52c-6efc-4516-a242-dfbc5c89d312","step_id":"61a4d52c-6efc-4516-a242-dfbc5c89d312","response_status":"success","response_out_args":{"__test__":{"type":"string","name":"__wrong_name__","value":"updated!"}}}` - bytesPayload := []byte(jsonPayload) - - expectedErr := "failed to unmarshal JSON: json: unknown field \"invalidProperty\"" - _, err := manualHandler.parseManualOutArgsUpdate(bytesPayload) - if err == nil { - t.Log("an error for non-matching variables names should have been raised") - t.Fail() - } - - assert.Equal(t, err.Error(), expectedErr) -} - -func TestParseCommandInfoToResponse(t *testing.T) { - - manualHandler := NewManualHandler(&mock_interaction_storage.MockInteractionStorage{}) - - testExecId := "50b6d52c-6efc-4516-a242-dfbc5c89d421" - testStepId := "61a4d52c-6efc-4516-a242-dfbc5c89d312" - testPlaybookId := "21a4d52c-6efc-4516-a242-dfbc5c89d312" - - command := cacao.Command{Type: "manual", Command: "please do a test thanks", Description: "testing!"} - target := cacao.AgentTarget{Type: "target", Name: "myself"} - variable2 := cacao.Variable{Type: "string", Name: "__test__", Value: "some value"} - inputVariable := map[string]cacao.Variable{"__test__": variable2} - - context := capability.Context{ - Command: command, - Target: target, - Variables: inputVariable, - } - - testVariables := cacao.NewVariables(cacao.Variable{Type: "string", Name: "__test__", Value: "test!"}) - - commandInfo := manual.CommandInfo{ - Metadata: execution.Metadata{ - PlaybookId: testPlaybookId, - ExecutionId: uuid.MustParse(testExecId), - StepId: testStepId}, - Context: context, - OutArgsVariables: testVariables, - } - - expectedInteractionCommand := api.InteractionCommandData{ - Type: "manual-command-info", - ExecutionId: testExecId, - PlaybookId: testPlaybookId, - StepId: testStepId, - Description: "testing!", - Command: "please do a test thanks", - CommandIsBase64: false, - Target: target, - OutVariables: testVariables, - } - - returnInteractionCommandData := manualHandler.parseCommandInfoToResponse(commandInfo) - t.Log(returnInteractionCommandData) - t.Log(expectedInteractionCommand) - - assert.Equal(t, reflect.DeepEqual(returnInteractionCommandData, expectedInteractionCommand), true) -} - -func TestParseManualOutArgsToInteractionResponse(t *testing.T) { - manualHandler := NewManualHandler(&mock_interaction_storage.MockInteractionStorage{}) - - testExecId := "50b6d52c-6efc-4516-a242-dfbc5c89d421" - testStepId := "61a4d52c-6efc-4516-a242-dfbc5c89d312" - testPlaybookId := "21a4d52c-6efc-4516-a242-dfbc5c89d312" - - outVariable := cacao.Variable{Type: "string", Name: "__test__", Value: "updated!"} - outVariables := map[string]cacao.Variable{"__test__": outVariable} - - payload := api.ManualOutArgsUpdatePayload{ - Type: "out-args-update", - ExecutionId: testExecId, - PlaybookId: testPlaybookId, - StepId: testStepId, - ResponseStatus: manual.ManualResponseFailureStatus, - ResponseOutArgs: outVariables, - } - - expetedInteractionResponse := manual.InteractionResponse{ - Metadata: execution.Metadata{ - PlaybookId: testPlaybookId, - ExecutionId: uuid.MustParse(testExecId), - StepId: testStepId, - }, - ResponseStatus: manual.ManualResponseFailureStatus, - OutArgsVariables: outVariables, - ResponseError: nil, - } - - interactionResponse, err := manualHandler.parseManualOutArgsToInteractionResponse(payload) - if err != nil { - t.Log(err) - t.Fail() - } - - assert.Equal(t, expetedInteractionResponse, interactionResponse) -} - -func TestParseManualOutArgsToInteractionResponseFailOnNonUUID(t *testing.T) { - manualHandler := NewManualHandler(&mock_interaction_storage.MockInteractionStorage{}) - - testExecId := "invalidUUID! 50b6d52c-6efc-4516-a242-dfbc5c89d421" - testStepId := "61a4d52c-6efc-4516-a242-dfbc5c89d312" - testPlaybookId := "21a4d52c-6efc-4516-a242-dfbc5c89d312" - - outVariable := cacao.Variable{Type: "string", Name: "__test__", Value: "updated!"} - outVariables := map[string]cacao.Variable{"__test__": outVariable} - - payload := api.ManualOutArgsUpdatePayload{ - Type: "out-args-update", - ExecutionId: testExecId, - PlaybookId: testPlaybookId, - StepId: testStepId, - ResponseStatus: manual.ManualResponseFailureStatus, - ResponseOutArgs: outVariables, - } - - expectedErr := "invalid UUID length: 49" - _, err := manualHandler.parseManualOutArgsToInteractionResponse(payload) - if err == nil { - t.Log(err) - t.Fail() - } - - assert.Equal(t, err.Error(), expectedErr) -} diff --git a/pkg/api/middelware/gin_log_middleware.go b/pkg/api/middelware/gin_log_middleware.go deleted file mode 100644 index 086720c6..00000000 --- a/pkg/api/middelware/gin_log_middleware.go +++ /dev/null @@ -1,33 +0,0 @@ -package loggerfactory - -import ( - "time" - - gin "github.com/gin-gonic/gin" - logrus "github.com/sirupsen/logrus" -) - - -func LoggingMiddleware(fl *logrus.Logger) gin.HandlerFunc { - return func(ctx *gin.Context) { - - startTime := time.Now() - ctx.Next() - endTime := time.Now() - latencyTime := endTime.Sub(startTime) - reqMethod := ctx.Request.Method - reqUri := ctx.Request.RequestURI - statusCode := ctx.Writer.Status() - clientIP := ctx.ClientIP() - - fl.WithFields(logrus.Fields{ - "METHOD": reqMethod, - "URI": reqUri, - "STATUS": statusCode, - "LATENCY": latencyTime, - "CLIENT_IP": clientIP, - }).Info("HTTP REQUEST") - - ctx.Next() - } -} \ No newline at end of file diff --git a/pkg/api/playbook/playbook_api.go b/pkg/api/playbook/playbook_api.go deleted file mode 100644 index e8b108ef..00000000 --- a/pkg/api/playbook/playbook_api.go +++ /dev/null @@ -1,200 +0,0 @@ -package playbook - -import ( - "io" - "net/http" - "reflect" - "soarca/internal/controller/database" - "soarca/internal/logger" - "strconv" - - playbookrepository "soarca/internal/database/playbook" - - "github.com/gin-gonic/gin" -) - -var log *logger.Log - -type Empty struct{} - -func init() { - log = logger.Logger(reflect.TypeOf(Empty{}).PkgPath(), logger.Info, "", logger.Json) -} - -// a playbookHandler implements the playbook api endpoints is dependent on a database. -type playbookHandler struct { - playbookRepo playbookrepository.IPlaybookRepository -} - -// NewPlaybookHandler makes a new instance of NewPlaybookHandler -func NewPlaybookHandler(controller database.IController) *playbookHandler { - return &playbookHandler{playbookRepo: controller.GetDatabaseInstance()} -} - -// GetAllPlaybooks GET handler for obtaining all the playbooks in the database and return this to the gin context in json format -// -// @Summary gets all the UUIDs for the stored playbooks -// @Schemes -// @Description return all stored playbooks default limit:100 -// @Tags playbook -// @Produce json -// @success 200 {array} cacao.Playbook -// @failure 400 {object} api.Error -// @Router /playbook/ [GET] -func (handler *playbookHandler) GetAllPlaybooks(g *gin.Context) { - log.Trace("Trying to obtain all playbook IDs") - - returnListIDs, err := handler.playbookRepo.GetPlaybooks() - if err != nil { - log.Debug("Could not obtain any PlaybookMetas", err) - SendErrorResponse(g, http.StatusBadRequest, "Could not obtain any IDs", "GET /playbook/meta") - return - } - - g.JSON(http.StatusOK, returnListIDs) -} - -// GetAllPlaybookMetas GET handler for obtaining all the meta data of all the stored playbooks -// in the database and return this to the gin context in json format -// -// @Summary gets all the meta information for the stored playbooks -// @Schemes -// @Description get playbook meta information for playbook -// @Tags playbook -// @Produce json -// @success 200 {array} api.PlaybookMeta -// @failure 400 {object} api.Error -// @Router /playbook/meta [GET] -func (handler *playbookHandler) GetAllPlaybookMetas(g *gin.Context) { - log.Trace("Trying to obtain all playbook IDs") - - returnListIDs, err := handler.playbookRepo.GetPlaybookMetas() - if err != nil { - log.Debug("Could not obtain any PlaybookMetas", err) - SendErrorResponse(g, http.StatusBadRequest, "Could not obtain any IDs", "GET /playbook/meta") - return - } - - g.JSON(http.StatusOK, returnListIDs) -} - -// SubmitPlaybook POST handler for creating playbooks. -// -// @Summary submit playbook via the api -// @Schemes -// @Description submit a new playbook api -// @Tags playbook -// @Produce json -// @Accept json -// @Param data body cacao.Playbook true "playbook" -// @Success 200 {object} cacao.Playbook -// @failure 400 {object} api.Error -// @Router /playbook/ [POST] -func (handler *playbookHandler) SubmitPlaybook(g *gin.Context) { - jsonData, err := io.ReadAll(g.Request.Body) - if err != nil { - log.Trace("Submit playbook Endpoint has failed: ", err.Error()) - SendErrorResponse(g, http.StatusBadRequest, "Failed to marshall json on server side", "POST /playbook") - return - } - playbook, err := handler.playbookRepo.Create(&jsonData) - if err != nil { - log.Debug("Submit playbook Endpoint has failed:", err.Error()) - if err.Error() == "duplicate" { - SendErrorResponse(g, http.StatusConflict, "Provided duplicate playbook, already in database", "POST /playbook") - } else { - SendErrorResponse(g, http.StatusBadRequest, "Could not create playbook. Is the playbook correct?", "POST /playbook") - } - return - } - g.JSON(http.StatusCreated, playbook) -} - -// GetPlaybookByID GET handler that finds playbook by id -// -// @Summary get CACAO playbook by its ID -// @Schemes -// @Description get playbook by ID -// @Tags playbook -// @Produce json -// @Accept json -// @Param id path string true "playbook ID" -// @Success 200 {object} cacao.Playbook -// @failure 400 {object} api.Error -// @Router /playbook/{id} [GET] -func (handler *playbookHandler) GetPlaybookByID(g *gin.Context) { - id := g.Param("id") - log.Trace("Trying to obtain playbook for id: ", id) - - playbook, err := handler.playbookRepo.Read(id) - if err != nil { - log.Debug("Could not find document for given id") - SendErrorResponse(g, http.StatusBadRequest, "Could not find playbook for given ID", "GET /playbook/{id}") - return - } - g.JSON(http.StatusOK, playbook) -} - -// UpdatePlaybookByID PUT handler that allows updating playbook object by ID. -// -// @Summary update playbook -// @Schemes -// @Description update playbook by Id -// @Tags playbook -// @Produce json -// @Accept json -// @Param id path string true "playbook Id" -// @Param data body cacao.Playbook true "playbook" -// @Success 200 {object} cacao.Playbook -// @failure 400 {object} api.Error -// @Router /playbook/{id} [PUT] -func (handler *playbookHandler) UpdatePlaybookByID(g *gin.Context) { - id := g.Param("id") - log.Trace("Trying to update playbook for id: ", id) - - jsonData, err := io.ReadAll(g.Request.Body) - if err != nil { - log.Debug("Update playbook Endpoint has failed: ", err.Error()) - SendErrorResponse(g, http.StatusBadRequest, "Failed to marshall json on server sider", "PUT /playbook/{id}") - return - } - updatedData, err := handler.playbookRepo.Update(id, &jsonData) - if err != nil { - log.Trace("Could not find document for given ") - SendErrorResponse(g, http.StatusBadRequest, "Could not find playbook for given ID", "PUT /playbook/{id}") - return - } - g.JSON(http.StatusOK, updatedData) -} - -// DeleteByPlaybookID DELETE handler for deleting playbook by ID. -// -// @Summary delete playbook by Id -// @Schemes -// @Description delete playbook by Id -// @Tags playbook -// @Produce json -// @Accept json -// @Param id path string true "playbook ID" -// @Success 200 -// @failure 400 {object} api.Error -// @Router /playbook/{id} [DELETE] -func (handler *playbookHandler) DeleteByPlaybookID(g *gin.Context) { - id := g.Param("id") - err := handler.playbookRepo.Delete(id) - if err != nil { - log.Debug("Something when wrong tying to delete the playbook object. Does the object exists?") - SendErrorResponse(g, http.StatusBadRequest, "Could not delete object", "DELETE /playbook/{id}") - return - } - g.Status(http.StatusOK) -} - -func SendErrorResponse(g *gin.Context, status int, message string, orginal_call string) { - msg := gin.H{ - "status": strconv.Itoa(status), - "message": message, - "original-call": orginal_call, - } - g.JSON(status, msg) -} diff --git a/pkg/api/reporter/reporter_api.go b/pkg/api/reporter/reporter_api.go deleted file mode 100644 index 682f123e..00000000 --- a/pkg/api/reporter/reporter_api.go +++ /dev/null @@ -1,105 +0,0 @@ -package reporter - -import ( - "net/http" - "reflect" - "soarca/internal/controller/informer" - "soarca/internal/logger" - "soarca/pkg/api/error" - "soarca/pkg/models/api" - - "github.com/gin-gonic/gin" - "github.com/google/uuid" -) - -var log *logger.Log - -type Empty struct{} - -func init() { - log = logger.Logger(reflect.TypeOf(Empty{}).PkgPath(), logger.Info, "", logger.Json) -} - -// reportHandler implements the handler functions that can be called by the gin api is dependent on a database. -type reportHandler struct { - informer informer.IExecutionInformer -} - -// NewReportHandler makes a new instance of playbookControler -func NewReportHandler(informer informer.IExecutionInformer) *reportHandler { - return &reportHandler{informer: informer} -} - -// GetExecutions GET handler for obtaining all the executions that can be retrieved. -// Returns this to the gin context as a list if execution IDs in json format -// -// @Summary gets all the UUIDs for the executions that can be retireved -// @Schemes -// @Description return all stored executions -// @Tags reporter -// @Produce json -// @success 200 {array} api.PlaybookExecutionReport -// @failure 400 {object} api.Error -// @Router /reporter [GET] -func (reportHandler *reportHandler) GetExecutions(g *gin.Context) { - executions, err := reportHandler.informer.GetExecutions() - if err != nil { - log.Debug("Could not get executions from informer") - error.SendErrorResponse(g, http.StatusInternalServerError, "Could not get executions from informer", "GET /reporter/", "") - return - } - - executionsParsed := []api.PlaybookExecutionReport{} - for _, executionEntry := range executions { - executionEntryParsed, err := parseCachePlaybookEntry(executionEntry) - if err != nil { - log.Debug("Could not parse entry to reporter result model") - log.Error(err) - error.SendErrorResponse(g, http.StatusInternalServerError, "Could not parse execution report", "GET /reporter/", "") - return - } - executionsParsed = append(executionsParsed, executionEntryParsed) - } - - g.JSON(http.StatusOK, executionsParsed) -} - -// GetExecutionReport GET handler for obtaining the information about an execution. -// Returns this to the gin context as a PlaybookExecutionReport object at soarca/model/api/reporter -// -// @Summary gets information about an ongoing playbook execution -// @Schemes -// @Description return execution information -// @Tags reporter -// @Produce json -// @Param id path string true "execution identifier" -// @success 200 {object} api.PlaybookExecutionReport -// @failure 400 {object} api.Error -// @Router /reporter/{id} [GET] -func (handler *reportHandler) GetExecutionReport(g *gin.Context) { - id := g.Param("id") - log.Trace("Trying to obtain execution for id: ", id) - uuid, err := uuid.Parse(id) - if err != nil { - log.Debug("Could not parse id parameter for request") - error.SendErrorResponse(g, http.StatusBadRequest, "Could not parse id parameter for request", "GET /reporter/"+id, err.Error()) - return - } - - executionEntry, err := handler.informer.GetExecutionReport(uuid) - if err != nil { - log.Debug("Could not find execution for given id") - log.Error(err) - error.SendErrorResponse(g, http.StatusBadRequest, "Could not find execution for given ID", "GET /reporter/"+id, "") - return - } - - executionEntryParsed, err := parseCachePlaybookEntry(executionEntry) - if err != nil { - log.Debug("Could not parse entry to reporter result model") - log.Error(err) - error.SendErrorResponse(g, http.StatusInternalServerError, "Could not parse execution report", "GET /reporter/"+id, "") - return - } - g.JSON(http.StatusOK, executionEntryParsed) -} diff --git a/pkg/api/reporter/reporter_parser.go b/pkg/api/reporter/reporter_parser.go deleted file mode 100644 index 7fa3c966..00000000 --- a/pkg/api/reporter/reporter_parser.go +++ /dev/null @@ -1,73 +0,0 @@ -package reporter - -import ( - api_model "soarca/pkg/models/api" - cache_model "soarca/pkg/models/cache" -) - -const defaultRequestInterval int = 5 - -func parseCachePlaybookEntry(cacheEntry cache_model.ExecutionEntry) (api_model.PlaybookExecutionReport, error) { - playbookStatus := api_model.CacheStatusEnum2String(cacheEntry.Status) - - playbookStatusText, err := api_model.GetCacheStatusText(playbookStatus, api_model.ReportLevelPlaybook) - if err != nil { - return api_model.PlaybookExecutionReport{}, err - } - if cacheEntry.Error != nil { - playbookStatusText = playbookStatusText + " - error: " + cacheEntry.Error.Error() - } - - stepResults, err := parseCacheStepEntries(cacheEntry.StepResults) - if err != nil { - return api_model.PlaybookExecutionReport{}, err - } - - executionReport := api_model.PlaybookExecutionReport{ - Type: "execution_status", - Name: cacheEntry.Name, - Description: cacheEntry.Description, - ExecutionId: cacheEntry.ExecutionId.String(), - PlaybookId: cacheEntry.PlaybookId, - Started: cacheEntry.Started, - Ended: cacheEntry.Ended, - Status: playbookStatus, - StatusText: playbookStatusText, - StepResults: stepResults, - RequestInterval: defaultRequestInterval, - } - return executionReport, nil -} - -func parseCacheStepEntries(cacheStepEntries map[string]cache_model.StepResult) (map[string]api_model.StepExecutionReport, error) { - parsedEntries := map[string]api_model.StepExecutionReport{} - for stepId, stepEntry := range cacheStepEntries { - - stepStatus := api_model.CacheStatusEnum2String(stepEntry.Status) - - stepStatusText, err := api_model.GetCacheStatusText(stepStatus, api_model.ReportLevelStep) - if err != nil { - return map[string]api_model.StepExecutionReport{}, err - } - - if stepEntry.Error != nil { - stepStatusText = stepStatusText + " - error: " + stepEntry.Error.Error() - } - - parsedEntries[stepId] = api_model.StepExecutionReport{ - ExecutionId: stepEntry.ExecutionId.String(), - StepId: stepEntry.StepId, - Name: stepEntry.Name, - Description: stepEntry.Description, - Started: stepEntry.Started, - Ended: stepEntry.Ended, - Status: stepStatus, - StatusText: stepStatusText, - ExecutedBy: "soarca", - CommandsB64: stepEntry.CommandsB64, - Variables: stepEntry.Variables, - AutomatedExecution: stepEntry.IsAutomated, - } - } - return parsedEntries, nil -} diff --git a/pkg/api/trigger/trigger_api.go b/pkg/api/trigger/trigger_api.go deleted file mode 100644 index f5d9ca5c..00000000 --- a/pkg/api/trigger/trigger_api.go +++ /dev/null @@ -1,192 +0,0 @@ -package trigger - -import ( - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "reflect" - "soarca/internal/controller/database" - "soarca/internal/controller/decomposer_controller" - "soarca/internal/logger" - "soarca/pkg/core/decomposer" - "soarca/pkg/models/api" - "soarca/pkg/models/cacao" - "soarca/pkg/models/decoder" - "time" - - apiError "soarca/pkg/api/error" - - "github.com/gin-gonic/gin" -) - -type Empty struct{} - -var log *logger.Log - -type ITrigger interface { - Execute(context *gin.Context) -} - -func init() { - log = logger.Logger(reflect.TypeOf(Empty{}).PkgPath(), logger.Info, "", logger.Json) -} - -type TriggerHandler struct { - controller decomposer_controller.IController - database database.IController - ExecutionsChannel chan decomposer.ExecutionDetails -} - -func NewTriggerHandler(controller decomposer_controller.IController, database database.IController) *TriggerHandler { - instance := TriggerHandler{} - instance.controller = controller - instance.database = database - // Channel to get back execution details - instance.ExecutionsChannel = make(chan decomposer.ExecutionDetails) - return &instance -} - -// trigger -// -// @Summary trigger a playbook by id that is stored in SOARCA -// @Schemes -// @Description trigger playbook by id -// @Tags trigger -// @Accept json -// @Produce json -// @Param id path string true "playbook ID" -// @Param data body cacao.Variables true "playbook" -// @Success 200 {object} api.Execution -// @failure 400 {object} api.Error -// @Router /trigger/playbook/{id} [POST] -func (handler *TriggerHandler) ExecuteById(context *gin.Context) { - log.Trace("received execute by ID") - id := context.Param("id") - - db := handler.database.GetDatabaseInstance() - playbook, err := db.Read(id) - if err != nil { - log.Error("failed to load playbook") - apiError.SendErrorResponse(context, http.StatusBadRequest, - "Failed to load playbook", - "POST /trigger/playbook/"+id, err.Error()) - return - } - if context.Request.Body != nil { - jsonData, err := io.ReadAll(context.Request.Body) - if err != nil { - log.Trace("Playbook trigger has failed to decode request body") - apiError.SendErrorResponse(context, http.StatusBadRequest, "Failed to decode request body", "POST /trigger/playbook/"+id, "") - } - err = MergeVariablesInPlaybook(&playbook, jsonData) - if err != nil { - log.Error(err) - apiError.SendErrorResponse(context, http.StatusBadRequest, fmt.Sprintf("Cannot execute. reason: %s", err), "POST /trigger/playbook/"+id, "") - return - } - } - handler.executePlaybook(&playbook, context) -} - -// trigger -// -// @Summary trigger a playbook by supplying a cacao playbook payload -// @Schemes -// @Description trigger playbook -// @Tags trigger -// @Accept json -// @Produce json -// @Param playbook body cacao.Playbook true "execute playbook by payload" -// @Success 200 {object} api.Execution -// @failure 400 {object} api.Error -// @Router /trigger/playbook [POST] -func (handler *TriggerHandler) Execute(context *gin.Context) { - log.Trace("received execute with body") - jsonData, err := io.ReadAll(context.Request.Body) - if err != nil { - log.Error("failed") - apiError.SendErrorResponse(context, http.StatusBadRequest, - "Failed to marshall json on server side", - "POST /trigger/playbook", "") - return - } - playbook := decoder.DecodeValidate(jsonData) - if playbook == nil { - log.Error("Failed to decode playbook") - apiError.SendErrorResponse(context, http.StatusBadRequest, - "Failed to decode playbook", - "POST /trigger/playbook", "") - return - } - - handler.executePlaybook(playbook, context) -} - -func (handler *TriggerHandler) executePlaybook(playbook *cacao.Playbook, context *gin.Context) { - decomposer := handler.controller.NewDecomposer() - go decomposer.ExecuteAsync(*playbook, handler.ExecutionsChannel) - timer := time.NewTimer(time.Duration(3) * time.Second) - for { - select { - case <-timer.C: - log.Error("async execution timed out for playbook ", playbook.ID) - - apiError.SendErrorResponse(context, - http.StatusRequestTimeout, - "async execution timed out for playbook "+playbook.ID, - "POST "+context.Request.URL.Path, "") - return - - case executionsDetail := <-handler.ExecutionsChannel: - playbookId := executionsDetail.PlaybookId - executionId := executionsDetail.ExecutionId - if playbookId == playbook.ID { - context.JSON(http.StatusOK, - api.Execution{ - ExecutionId: executionId, - PlaybookId: playbookId, - }) - return - } - } - } -} - -// public fun as tested externally (integration test) -func MergeVariablesInPlaybook(playbook *cacao.Playbook, body []byte) error { - payloadVariables := cacao.NewVariables() - err := json.Unmarshal(body, &payloadVariables) - if err != nil { - log.Trace(err) - return errors.New("cannot unmarshal provided variables") - } - - // Check payload-injected variables are valid set for playbook variables - for name, variable := range payloadVariables { - // Must exist - if _, ok := playbook.PlaybookVariables[name]; !ok { - return fmt.Errorf("provided variables is not a valid subset of the variables for the referenced playbook [ playbook id: %s ]", playbook.ID) - } - // Exists, playbook var type must match - if variable.Type != playbook.PlaybookVariables[name].Type { - return fmt.Errorf("mismatch in variables type for [ %s ]: payload var type = %s, playbook var type = %s", name, variable.Type, playbook.PlaybookVariables[name].Type) - } - // Exists, playbook var must be external - if !playbook.PlaybookVariables[name].External { - return fmt.Errorf("playbook variable [ %s ] cannot be assigned in playbook because it is not marked as external in the plabook", name) - } - - updatedVariable := cacao.Variable{ - Name: name, - Type: playbook.PlaybookVariables[name].Type, - Description: playbook.PlaybookVariables[name].Description, - Value: variable.Value, - Constant: playbook.PlaybookVariables[name].Constant, - External: playbook.PlaybookVariables[name].External, - } - playbook.PlaybookVariables[name] = updatedVariable - } - return nil -} diff --git a/pkg/models/cacao/initializers.go b/pkg/cacao/initializers.go similarity index 100% rename from pkg/models/cacao/initializers.go rename to pkg/cacao/initializers.go diff --git a/pkg/models/cacao/cacao.go b/pkg/cacao/playbook.go similarity index 100% rename from pkg/models/cacao/cacao.go rename to pkg/cacao/playbook.go diff --git a/pkg/models/cacao/variables.go b/pkg/cacao/variables.go similarity index 100% rename from pkg/models/cacao/variables.go rename to pkg/cacao/variables.go diff --git a/pkg/models/cacao/variables_test.go b/pkg/cacao/variables_test.go similarity index 100% rename from pkg/models/cacao/variables_test.go rename to pkg/cacao/variables_test.go diff --git a/pkg/core/capability/capability.go b/pkg/core/capability/capability.go deleted file mode 100644 index 25eeb72d..00000000 --- a/pkg/core/capability/capability.go +++ /dev/null @@ -1,20 +0,0 @@ -package capability - -import ( - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" -) - -type Context struct { - Command cacao.Command - Step cacao.Step - Authentication cacao.AuthenticationInformation - Target cacao.AgentTarget - Variables cacao.Variables -} - -type ICapability interface { - Execute(metadata execution.Metadata, - context Context) (cacao.Variables, error) - GetType() string -} diff --git a/pkg/core/capability/fin/controller/controller.go b/pkg/core/capability/fin/controller/controller.go deleted file mode 100644 index 7d75fe69..00000000 --- a/pkg/core/capability/fin/controller/controller.go +++ /dev/null @@ -1,191 +0,0 @@ -package controller - -import ( - "errors" - "fmt" - "reflect" - "soarca/internal/logger" - "soarca/pkg/core/capability/fin/protocol" - "soarca/pkg/models/fin" - - mqtt "github.com/eclipse/paho.mqtt.golang" -) - -type Empty struct{} - -var component = reflect.TypeOf(Empty{}).PkgPath() -var log *logger.Log - -func init() { - log = logger.Logger(component, logger.Info, "", logger.Json) -} - -type CapabilityDetails struct { - Name string - Id string - FinId string -} - -const clientId = "soarca" - -type IFinController interface { - GetRegisteredCapabilities() map[string]CapabilityDetails -} - -type FinController struct { - registeredCapabilities map[string]CapabilityDetails - mqttClient mqtt.Client - channel chan []byte -} - -func (finController *FinController) GetRegisteredCapabilities() map[string]CapabilityDetails { - return finController.registeredCapabilities -} - -func New(client mqtt.Client) *FinController { - controllerQueue := make(chan []byte, 10) - return &FinController{registeredCapabilities: make(map[string]CapabilityDetails), mqttClient: client, channel: controllerQueue} -} - -func NewClient(url protocol.Broker, port int) *mqtt.Client { - options := mqtt.NewClientOptions() - options.AddBroker(fmt.Sprintf("mqtt://%s:%d", url, port)) - options.SetClientID(clientId) - options.SetUsername("soarca") - options.SetPassword("password") - client := mqtt.NewClient(options) - return &client -} - -func (finController *FinController) ConnectAndSubscribe() error { - if finController.mqttClient == nil { - return errors.New("fincontroller mqtt cilent is nil") - } - - if token := finController.mqttClient.Connect(); token.Wait() && token.Error() != nil { - err := token.Error() - log.Error(err) - return err - } - - token := finController.mqttClient.Subscribe(string("soarca"), 1, finController.Handler) - token.Wait() - if err := token.Error(); err != nil { - return err - } - return nil -} - -// This function will only return on a fatal error -func (finController *FinController) Run() { - for { - result := <-finController.channel - finController.Handle(result) - } -} - -// Handle goroutine call from mqtt stack -func (finController *FinController) Handler(client mqtt.Client, msg mqtt.Message) { - // Might need to filter on fin topics in the future if communication is needed - if msg.Topic() != string("soarca") { - log.Trace("message was receive in wrong topic: " + msg.Topic()) - return - } - payload := msg.Payload() - log.Trace(string(payload)) - finController.channel <- payload - -} - -func (finController *FinController) SendAck(topic string, messageId string) error { - json, _ := fin.Encode(fin.NewAck(messageId)) - log.Trace("Sending ack for message id: ", messageId) - token := finController.mqttClient.Publish(topic, 1, false, json) - token.Wait() - if err := token.Error(); err != nil { - log.Error(err) - return err - } - return nil -} - -func (finController *FinController) Handle(payload []byte) { - message := fin.Message{} - if err := fin.Decode(payload, &message); err != nil { - log.Error(err) - return - } - switch message.Type { - case fin.MessageTypeAck: - finController.HandleAck(payload) - case fin.MessageTypeRegister: - finController.HandleRegister(payload) - case fin.MessageTypeNack: - finController.HandleNack(payload) - } -} - -func (finController *FinController) SendNack(topic string, messageId string) error { - json, _ := fin.Encode(fin.NewNack(messageId)) - log.Trace("Sending nack for message id: ", messageId) - token := finController.mqttClient.Publish(topic, 1, false, json) - token.Wait() - if err := token.Error(); err != nil { - log.Error(err) - return err - } - return nil -} - -func (finController *FinController) HandleAck(payload []byte) { - ack := fin.Ack{} - if err := fin.Decode(payload, ack); err != nil { - log.Error(err) - } - - // ignore for now - -} - -func (finController *FinController) HandleNack(payload []byte) { - ack := fin.Ack{} - if err := fin.Decode(payload, ack); err != nil { - log.Error(err) - } - - // ignore for now - -} - -func (finController *FinController) HandleRegister(payload []byte) { - register := fin.Register{} - err := fin.Decode(payload, ®ister) - if err != nil { - log.Error("Message", err) - if err := finController.SendNack("soarca", register.MessageId); err != nil { - log.Error(err) - } - return - } - - for _, capability := range register.Capabilities { - if _, ok := finController.registeredCapabilities[capability.Id]; ok { - if err := finController.SendNack(register.FinID, register.MessageId); err != nil { - log.Error(err) - } - log.Error("this capability UUID is already registered") - return - } - token := finController.mqttClient.Subscribe(capability.Id, 1, finController.Handler) - token.Wait() - - detail := CapabilityDetails{Name: capability.Name, Id: capability.Id, FinId: register.FinID} - finController.registeredCapabilities[capability.Id] = detail - - } - - if err := finController.SendAck(register.FinID, register.MessageId); err != nil { - log.Error(err) - } - -} diff --git a/pkg/core/capability/fin/controller/controller_test.go b/pkg/core/capability/fin/controller/controller_test.go deleted file mode 100644 index 725b5e79..00000000 --- a/pkg/core/capability/fin/controller/controller_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package controller - -import ( - "encoding/json" - "soarca/pkg/models/fin" - "soarca/test/unittest/mocks/mock_mqtt" - "testing" - - "github.com/go-playground/assert/v2" - "github.com/google/uuid" - "github.com/stretchr/testify/mock" -) - -func TestGetRegisteredc(t *testing.T) { - mqtt := new(mock_mqtt.Mock_MqttClient) - token := mock_mqtt.Mock_MqttToken{} - token2 := mock_mqtt.Mock_MqttToken{} - capabiltyController := New(mqtt) - fins := capabiltyController.GetRegisteredCapabilities() - - assert.Equal(t, len(fins), 0) - - messageId := uuid.New() - - capability := fin.Capability{Name: "cap1", Id: "id1", Version: "1.0.0"} - capabilities := make([]fin.Capability, 0) - capabilities = append(capabilities, capability) - - meta := fin.Meta{} - - incommingRegisterMessage := fin.Register{Type: fin.MessageTypeRegister, - MessageId: messageId.String(), - FinID: "Fin", - ProtocolVersion: "1.0.0", - Security: fin.Security{Version: "0.0.0", ChannelSecurity: ""}, - Capabilities: capabilities, - Meta: meta, - } - - object, err := json.Marshal(incommingRegisterMessage) - if err != nil { - t.Fail() - } - - token.On("Wait").Return(true) - mqtt.On("Subscribe", "id1", uint8(1), mock.Anything).Return(&token) - - expectedAck := fin.NewAck(messageId.String()) - json, _ := fin.Encode(expectedAck) - token2.On("Wait").Return(true) - mqtt.On("Publish", "Fin", uint8(1), false, json).Return(&token2) - token2.On("Error").Return(nil) - - capabiltyController.Handle(object) - - newFins := capabiltyController.GetRegisteredCapabilities() - - assert.Equal(t, len(newFins), 1) - assert.Equal(t, newFins["id1"].Id, "id1") - assert.Equal(t, newFins["id1"].Name, "cap1") - mqtt.AssertExpectations(t) - token.AssertExpectations(t) - token2.AssertExpectations(t) - -} - -func TestConnectAndSubsribe(t *testing.T) { - mqtt := new(mock_mqtt.Mock_MqttClient) - token := mock_mqtt.Mock_MqttToken{} - capabiltyController := New(mqtt) - - token.On("Wait").Return(true) - token.On("Error").Return(nil) - mqtt.On("Connect").Return(&token) - token.On("Wait").Return(true) - token.On("Error").Return(nil) - mqtt.On("Subscribe", "soarca", uint8(1), mock.Anything).Return(&token) - err := capabiltyController.ConnectAndSubscribe() - assert.Equal(t, err, nil) - mqtt.AssertExpectations(t) - token.AssertExpectations(t) -} diff --git a/pkg/core/capability/fin/fin.go b/pkg/core/capability/fin/fin.go deleted file mode 100644 index ad89a57c..00000000 --- a/pkg/core/capability/fin/fin.go +++ /dev/null @@ -1,46 +0,0 @@ -package fin - -import ( - "reflect" - "soarca/internal/logger" - "soarca/pkg/core/capability" - "soarca/pkg/core/capability/fin/protocol" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" - finModel "soarca/pkg/models/fin" -) - -type FinCapability struct { - finProtocol protocol.IFinProtocol -} - -var component = reflect.TypeOf(FinCapability{}).PkgPath() -var log *logger.Log - -func init() { - log = logger.Logger(component, logger.Info, "", logger.Json) -} - -func New(finProtocol protocol.IFinProtocol) *FinCapability { - return &FinCapability{finProtocol: finProtocol} -} - -func (FinCapability *FinCapability) GetType() string { - return "soarca-fin" -} - -func (finCapability *FinCapability) Execute( - metadata execution.Metadata, - context capability.Context) (cacao.Variables, error) { - - finCommand := finModel.NewCommand() - finCommand.CommandSubstructure.Command = context.Command.Command - finCommand.CommandSubstructure.Authentication = context.Authentication - finCommand.CommandSubstructure.Variables = context.Variables - finCommand.CommandSubstructure.Context.ExecutionId = metadata.ExecutionId.String() - finCommand.CommandSubstructure.Context.PlaybookId = metadata.PlaybookId - finCommand.CommandSubstructure.Context.StepId = metadata.StepId - - log.Trace("created command ", finCommand) - return finCapability.finProtocol.SendCommand(finCommand) -} diff --git a/pkg/core/capability/fin/fin_test.go b/pkg/core/capability/fin/fin_test.go deleted file mode 100644 index 05e7ce05..00000000 --- a/pkg/core/capability/fin/fin_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package fin - -import ( - "soarca/pkg/core/capability" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" - model "soarca/pkg/models/fin" - "soarca/test/unittest/mocks/mock_finprotocol" - "testing" - - "github.com/go-playground/assert/v2" - "github.com/google/uuid" -) - -func TestFinExecution(t *testing.T) { - mockFinProtocol := new(mock_finprotocol.MockFinProtocol) - //mockGuid := new(mock_guid.Mock_Guid) - finCapability := New(mockFinProtocol) - - var executionId, _ = uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") - var playbookId, _ = uuid.Parse("d09351a2-a075-40c8-8054-0b7c423db83f") - var stepId, _ = uuid.Parse("81eff59f-d084-4324-9e0a-59e353dbd28f") - - var metadata = execution.Metadata{ExecutionId: executionId, PlaybookId: playbookId.String(), StepId: stepId.String()} - - command := cacao.Command{Type: "soarca-fin", Command: "test command"} - auth := cacao.AuthenticationInformation{} - auth.Name = "some auth" - auth.Username = "user" - auth.Password = "password" - target := cacao.AgentTarget{} - variable1 := cacao.Variable{Type: "int", Name: "output", Value: "10"} - - //var id, _ = uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") - - expectedCommand := model.Command{} - expectedCommand.Type = "command" - expectedCommand.CommandSubstructure.Context.ExecutionId = executionId.String() - - expectedCommand.CommandSubstructure.Authentication = auth - expectedCommand.CommandSubstructure.Command = "test command" - expectedCommand.CommandSubstructure.Context.Timeout = 1 - expectedCommand.CommandSubstructure.Context.PlaybookId = playbookId.String() - expectedCommand.CommandSubstructure.Context.StepId = stepId.String() - - variable2 := cacao.Variable{Type: "string", Name: "input", Value: "some value"} - inputVariable := map[string]cacao.Variable{"input_variable": variable2} - expectedCommand.CommandSubstructure.Variables = inputVariable - - //expectedCommand.CommandSubstructure.Context.GeneratedOn = "" - - expectedVariableMap := cacao.NewVariables(variable1) - - data := capability.Context{ - Command: command, - Authentication: auth, - Target: target, - Variables: inputVariable, - } - - //mockGuid.On("New").Return(id) - mockFinProtocol.On("SendCommand", expectedCommand).Return(expectedVariableMap, nil) - result, err := finCapability.Execute(metadata, data) - - assert.Equal(t, err, nil) - assert.Equal(t, result, expectedVariableMap) - -} diff --git a/pkg/core/capability/fin/protocol/finprotocol_test.go b/pkg/core/capability/fin/protocol/finprotocol_test.go deleted file mode 100644 index 4979d9ab..00000000 --- a/pkg/core/capability/fin/protocol/finprotocol_test.go +++ /dev/null @@ -1,120 +0,0 @@ -package protocol - -import ( - "encoding/json" - "errors" - "fmt" - "soarca/pkg/models/cacao" - "soarca/pkg/models/fin" - "soarca/pkg/utils/guid" - "soarca/test/unittest/mocks/mock_mqtt" - "testing" - "time" - - "github.com/go-playground/assert/v2" - "github.com/stretchr/testify/mock" -) - -func TestSubscribe(t *testing.T) { - mock_client := mock_mqtt.Mock_MqttClient{} - mock_token := mock_mqtt.Mock_MqttToken{} - - guid := new(guid.Guid) - prot := FinProtocol{Guid: guid, Topic: Topic("testing"), Broker: "localhost", Port: 1883} - - mock_token.On("Wait").Return(true) - mock_client.On("Subscribe", "testing", uint8(1), mock.Anything).Return(&mock_token) - prot.Subscribe(&mock_client) - -} - -func TestTimeoutAndCallbackTimerElaspsed(t *testing.T) { - mock_client := mock_mqtt.Mock_MqttClient{} - mock_token := mock_mqtt.Mock_MqttToken{} - - guid := new(guid.Guid) - prot := FinProtocol{Guid: guid, Topic: Topic("testing"), Broker: "localhost", Port: 1883} - - mock_token.On("Wait").Return(true) - mock_client.On("Subscribe", "testing", uint8(1), mock.Anything).Return(&mock_token) - prot.Subscribe(&mock_client) - - expectedCommand := fin.NewCommand() - expectedCommand.CommandSubstructure.Context.Timeout = 1 - - result, err := prot.AwaitResultOrTimeout(expectedCommand, &mock_client) - - assert.Equal(t, err, errors.New("no message received from fin while it was expected")) - assert.Equal(t, result, cacao.NewVariables()) -} - -func TestTimeoutAndCallbackHandlerCalled(t *testing.T) { - mock_client := mock_mqtt.Mock_MqttClient{} - mock_token := mock_mqtt.Mock_MqttToken{} - - mock_token_ack := mock_mqtt.Mock_MqttToken{} - - guid := new(guid.Guid) - - prot := New(guid, "testing", "localhost", 1883) - mock_token.On("Wait").Return(true) - mock_client.On("Subscribe", "testing", uint8(1), mock.Anything).Return(&mock_token) - - prot.Subscribe(&mock_client) - - expectedCommand := fin.NewCommand() - expectedCommand.CommandSubstructure.Context.Timeout = 1 - - mock_token_ack.On("Wait").Return(true) - mock_client.On("Publish", "testing", uint8(1), false, mock.Anything).Return(&mock_token_ack) - - fmt.Println("calling await") - go helper(&prot) - result, err := prot.AwaitResultOrTimeout(expectedCommand, &mock_client) - fmt.Println("done waiting") - - assert.Equal(t, err, nil) - assert.Equal(t, result, cacao.NewVariables(cacao.Variable{Name: "test"})) - mock_client.AssertExpectations(t) - mock_token.AssertExpectations(t) - mock_token_ack.AssertExpectations(t) -} - -// Helper for TestTimeoutAndCallbackHandlerCalled -func helper(prot *FinProtocol) { - time.Sleep(1 * time.Millisecond) - client := mock_mqtt.Mock_MqttClient{} - message := mock_mqtt.Mock_MqttMessage{} - - ack := fin.Ack{} - ack.Type = fin.MessageTypeAck - ack.MessageId = "0001" - ackPayload, err := json.Marshal(ack) - if err != nil { - fmt.Print(err) - return - } - - message.On("Topic").Return("testing") - message.On("Payload").Return(ackPayload) - fmt.Println("calling handler") - prot.Handler(&client, &message) - - message2 := mock_mqtt.Mock_MqttMessage{} - - result := fin.Result{} - result.Type = fin.MessageTypeResult - result.ResultStructure.Variables = cacao.NewVariables(cacao.Variable{Name: "test"}) - - payload, err := json.Marshal(result) - if err != nil { - fmt.Print(err) - return - } - time.Sleep(1 * time.Millisecond) - message2.On("Topic").Return("testing") - message2.On("Payload").Return(payload) - prot.Handler(&client, &message2) - fmt.Println("called handler") - -} diff --git a/pkg/core/capability/fin/protocol/protocol.go b/pkg/core/capability/fin/protocol/protocol.go deleted file mode 100644 index 9c27ba78..00000000 --- a/pkg/core/capability/fin/protocol/protocol.go +++ /dev/null @@ -1,201 +0,0 @@ -package protocol - -import ( - "errors" - "fmt" - "reflect" - "soarca/internal/logger" - "soarca/pkg/models/cacao" - "soarca/pkg/models/fin" - "soarca/pkg/utils/guid" - "time" - - mqttlib "github.com/eclipse/paho.mqtt.golang" -) - -const defaultTimeout = 1 -const disconnectTimeout = 100 -const clientId = "soarca-fin-capability" -const defaultQos = AtLeastOnce - -const ( - AtMostOnce = iota - AtLeastOnce - ExactlyOnce -) - -type Topic string -type Message string -type Broker string - -var component = reflect.TypeOf(FinProtocol{}).PkgPath() -var log *logger.Log - -// var channel = make(chan []byte, 1) - -func init() { - log = logger.Logger(component, logger.Info, "", logger.Json) -} - -type IFinProtocol interface { - SendCommand(fin.Command) (cacao.Variables, error) -} - -type FinProtocol struct { - Topic Topic - Broker Broker - Port int - Guid guid.IGuid - channel chan []byte // Channel is for one instance and is private to this fin -} - -func New(guid guid.IGuid, topic Topic, broker Broker, port int) FinProtocol { - var channel = make(chan []byte, 1) - prot := FinProtocol{Guid: guid, Topic: topic, Broker: broker, Port: port, channel: channel} - return prot -} - -func (protocol *FinProtocol) SendAck(result fin.Result, client mqttlib.Client) { - ack := fin.NewAck(result.MessageId) - json, _ := fin.Encode(ack) - log.Trace("Sending ack for message id: ", result.MessageId) - token := client.Publish(string(protocol.Topic), defaultQos, false, json) - token.Wait() -} - -func (protocol *FinProtocol) SendNack(result fin.Result, client mqttlib.Client) { - nack := fin.NewNack(result.MessageId) - json, _ := fin.Encode(nack) - log.Trace("Sending nack for message id: ", result.MessageId) - token := client.Publish(string(protocol.Topic), defaultQos, false, json) - token.Wait() -} - -func (protocol *FinProtocol) SendCommand(command fin.Command) (cacao.Variables, error) { - - client, err := protocol.Connect(command.CommandSubstructure.Authentication) - if err != nil { - log.Error("could not connect to mqtt broker") - return nil, err - } - - protocol.Subscribe(client) - err = protocol.Publish(client, command) - if err != nil { - protocol.Disconnect(client) - return cacao.NewVariables(), err - } - result, err := protocol.AwaitResultOrTimeout(command, client) - protocol.Disconnect(client) - - return result, err -} - -func (protocol *FinProtocol) AwaitResultOrTimeout(command fin.Command, client mqttlib.Client) (cacao.Variables, error) { - timeout := command.CommandSubstructure.Context.Timeout - - if command.CommandSubstructure.Context.Timeout == 0 { - log.Warning("no valid timeout will set 1 second") - timeout = defaultTimeout - } - timer := time.NewTimer(time.Duration(timeout) * time.Second) - - // Wait in a loop for the timer to elapse or a message on the channel - ackReceived := false - - for { - select { - case <-timer.C: - err := errors.New("no message received from fin while it was expected") - return cacao.NewVariables(), err - case result := <-protocol.channel: - finMessage := fin.Message{} - err := fin.Decode(result, &finMessage) - if err != nil { - log.Trace(err) - break - } - log.Info(finMessage) - - // This now accepts any ack, should be changed - switch finMessage.Type { - case fin.MessageTypeAck: - ackReceived = true - case fin.MessageTypeResult: - finResult := fin.Result{} - err := fin.Decode(result, &finResult) - if err != nil { - log.Trace(err) - return cacao.NewVariables(), err - } - - if ackReceived { - - if finResult.ResultStructure.Context.ExecutionId == command.CommandSubstructure.Context.ExecutionId { - - protocol.SendAck(finResult, client) - return finResult.ResultStructure.Variables, nil - } else { - protocol.SendNack(finResult, client) - } - - } else { - protocol.SendNack(finResult, client) - } - } - } - - } - -} - -func (protocol *FinProtocol) Handler(client mqttlib.Client, msg mqttlib.Message) { - if msg.Topic() != string(protocol.Topic) { - log.Trace("message was receive in wrong topic: " + protocol.Topic) - } - payload := msg.Payload() - log.Trace(string(payload)) - protocol.channel <- payload - -} - -func (protocol *FinProtocol) Subscribe(client mqttlib.Client) { - token := client.Subscribe(string(protocol.Topic), defaultQos, protocol.Handler) - token.Wait() - -} - -func (protocol *FinProtocol) Publish(client mqttlib.Client, command fin.Command) error { - command.MessageId = protocol.Guid.New().String() - command.Meta.SenderId = clientId - command.Meta.Timestamp = time.Now() - - data, err := fin.Encode(command) - if err != nil { - return err - } - token := client.Publish(string(protocol.Topic), defaultQos, false, data) - token.Wait() - return token.Error() - -} - -func (protocol *FinProtocol) Connect(authenticationInformation cacao.AuthenticationInformation) (mqttlib.Client, error) { - options := mqttlib.NewClientOptions() - options.AddBroker(fmt.Sprintf("mqtt://%s:%d", protocol.Broker, protocol.Port)) - options.SetClientID(clientId) - options.SetUsername(authenticationInformation.Username) - options.SetPassword(authenticationInformation.Password) - - client := mqttlib.NewClient(options) - if token := client.Connect(); token.Wait() && token.Error() != nil { - err := token.Error() - log.Error(err) - return nil, err - } - return client, nil -} - -func (protocol *FinProtocol) Disconnect(client mqttlib.Client) { - client.Disconnect(disconnectTimeout) -} diff --git a/pkg/core/capability/http/http.go b/pkg/core/capability/http/http.go deleted file mode 100644 index e9133a21..00000000 --- a/pkg/core/capability/http/http.go +++ /dev/null @@ -1,65 +0,0 @@ -package http - -import ( - "reflect" - "soarca/internal/logger" - "soarca/pkg/core/capability" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" - "soarca/pkg/utils/http" -) - -// Receive HTTP API command data from decomposer/executer -// Validate HTTP API call -// Run HTTP API call -// Return response - -const ( - httpApiResultVariableName = "__soarca_http_api_result__" - httpApiCapabilityName = "soarca-http-api" -) - -type HttpCapability struct { - soarca_http_request http.IHttpRequest -} - -type Empty struct{} - -var component = reflect.TypeOf(Empty{}).PkgPath() -var log *logger.Log - -func init() { - log = logger.Logger(component, logger.Info, "", logger.Json) -} - -func New(httpRequest http.IHttpRequest) *HttpCapability { - return &HttpCapability{soarca_http_request: httpRequest} -} - -func (httpCapability *HttpCapability) GetType() string { - return httpApiCapabilityName -} - -func (httpCapability *HttpCapability) Execute( - metadata execution.Metadata, - context capability.Context) (cacao.Variables, error) { - - soarca_http_options := http.HttpOptions{ - Target: &context.Target, - Command: &context.Command, - Auth: &context.Authentication, - } - - responseBytes, err := httpCapability.soarca_http_request.Request(soarca_http_options) - if err != nil { - log.Error(err) - return cacao.NewVariables(), err - } - respString := string(responseBytes) - variable := cacao.Variable{Type: cacao.VariableTypeString, - Name: httpApiResultVariableName, - Value: respString} - - return cacao.NewVariables(variable), nil - -} diff --git a/pkg/core/capability/manual/interaction/interaction.go b/pkg/core/capability/manual/interaction/interaction.go deleted file mode 100644 index 3ba0c8b7..00000000 --- a/pkg/core/capability/manual/interaction/interaction.go +++ /dev/null @@ -1,264 +0,0 @@ -package interaction - -import ( - "context" - "errors" - "fmt" - "reflect" - "soarca/internal/logger" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" - "soarca/pkg/models/manual" -) - -type Empty struct{} - -var component = reflect.TypeOf(Empty{}).PkgPath() -var log *logger.Log - -func init() { - log = logger.Logger(component, logger.Info, "", logger.Json) -} - -type IInteractionIntegrationNotifier interface { - Notify(command manual.InteractionIntegrationCommand, channel chan manual.InteractionResponse) -} - -type ICapabilityInteraction interface { - Queue(command manual.CommandInfo, manualComms manual.ManualCapabilityCommunication) error -} - -type IInteractionStorage interface { - GetPendingCommands() ([]manual.CommandInfo, error) - // even if step has multiple manual commands, there should always be just one pending manual command per action step - GetPendingCommand(metadata execution.Metadata) (manual.CommandInfo, error) - PostContinue(response manual.InteractionResponse) error -} - -type InteractionController struct { - InteractionStorage map[string]map[string]manual.InteractionStorageEntry // Keyed on [executionID][stepID] - Notifiers []IInteractionIntegrationNotifier -} - -func New(manualIntegrations []IInteractionIntegrationNotifier) *InteractionController { - storage := map[string]map[string]manual.InteractionStorageEntry{} - return &InteractionController{ - InteractionStorage: storage, - Notifiers: manualIntegrations, - } -} - -// ############################################################################ -// ICapabilityInteraction implementation -// ############################################################################ -func (manualController *InteractionController) Queue(command manual.CommandInfo, manualComms manual.ManualCapabilityCommunication) error { - - err := manualController.registerPendingInteraction(command, manualComms.Channel) - if err != nil { - return err - } - - if _, ok := manualComms.TimeoutContext.Deadline(); !ok { - return errors.New("manual command does not have a deadline") - } - - // Copy and type conversion - integrationCommand := manual.InteractionIntegrationCommand(command) - - // One response channel for all integrations - integrationChannel := make(chan manual.InteractionResponse) - - for _, notifier := range manualController.Notifiers { - go notifier.Notify(integrationCommand, integrationChannel) - } - - // Async idle wait on command-specific channel closure - go manualController.handleManualCommandResponse(command, manualComms) - - return nil -} - -func (manualController *InteractionController) handleManualCommandResponse(command manual.CommandInfo, manualComms manual.ManualCapabilityCommunication) { - log.Trace( - fmt.Sprintf( - "goroutine handling command response %s, %s has started", command.Metadata.ExecutionId.String(), command.Metadata.StepId)) - defer log.Trace( - fmt.Sprintf( - "goroutine handling command response %s, %s has ended", command.Metadata.ExecutionId.String(), command.Metadata.StepId)) - - // Wait for either timeout or response - <-manualComms.TimeoutContext.Done() - if manualComms.TimeoutContext.Err() == context.DeadlineExceeded { - log.Info("manual command timed out. deregistering associated pending command") - } else if manualComms.TimeoutContext.Err() == context.Canceled { - log.Info("manual command completed. deregistering associated pending command") - } - err := manualController.removeInteractionFromPending(command.Metadata) - if err != nil { - log.Warning(err) - log.Warning("manual command not found among pending ones. should be already resolved") - return - } -} - -// ############################################################################ -// IInteractionStorage implementation -// ############################################################################ -func (manualController *InteractionController) GetPendingCommands() ([]manual.CommandInfo, error) { - log.Trace("getting pending manual commands") - return manualController.getAllPendingCommandsInfo(), nil -} - -func (manualController *InteractionController) GetPendingCommand(metadata execution.Metadata) (manual.CommandInfo, error) { - log.Trace("getting pending manual command") - interaction, err := manualController.getPendingInteraction(metadata) - return interaction.CommandInfo, err -} - -func (manualController *InteractionController) PostContinue(response manual.InteractionResponse) error { - log.Trace("completing manual command") - - // If not in there, it means it was already solved, or expired - pendingEntry, err := manualController.getPendingInteraction(response.Metadata) - if err != nil { - log.Warning(err) - return err - } - - warnings, err := manualController.validateMatchingOutArgs(pendingEntry, response.OutArgsVariables) - if err != nil { - return err - } - - //Then put outArgs back into manualCapabilityChannel - // Copy result and conversion back to interactionResponse format - log.Trace("pushing assigned variables in manual capability channel") - pendingEntry.Channel <- response - - if len(warnings) > 0 { - for _, warning := range warnings { - log.Warning(warning) - } - } - - return nil -} - -// ############################################################################ -// Utilities and functionalities -// ############################################################################ -func (manualController *InteractionController) registerPendingInteraction(command manual.CommandInfo, manualChan chan manual.InteractionResponse) error { - - commandInfo := manual.CommandInfo{ - Metadata: command.Metadata, - Context: command.Context, - OutArgsVariables: command.OutArgsVariables, - } - - execution, ok := manualController.InteractionStorage[commandInfo.Metadata.ExecutionId.String()] - - if !ok { - // It's fine, no entry for execution registered. Register execution and step entry - manualController.InteractionStorage[commandInfo.Metadata.ExecutionId.String()] = map[string]manual.InteractionStorageEntry{ - commandInfo.Metadata.StepId: { - CommandInfo: commandInfo, - Channel: manualChan, - }, - } - return nil - } - - // There is an execution entry - if _, ok := execution[commandInfo.Metadata.StepId]; ok { - // Error: there is already a pending manual command for the action step - err := fmt.Errorf( - "a manual step is already pending for execution %s, step %s. There can only be one pending manual command per action step", - commandInfo.Metadata.ExecutionId.String(), commandInfo.Metadata.StepId) - log.Error(err) - return err - } - - // Execution exist, and Finally register pending command in existing execution - // Question: is it ever the case that the same exact step is executed in parallel branches? Then this code would not work - execution[commandInfo.Metadata.StepId] = manual.InteractionStorageEntry{ - CommandInfo: commandInfo, - Channel: manualChan, - } - - return nil -} - -func (manualController *InteractionController) getAllPendingCommandsInfo() []manual.CommandInfo { - allPendingInteractions := []manual.CommandInfo{} - for _, interactions := range manualController.InteractionStorage { - for _, interaction := range interactions { - allPendingInteractions = append(allPendingInteractions, interaction.CommandInfo) - } - } - return allPendingInteractions -} - -func (manualController *InteractionController) getPendingInteraction(commandMetadata execution.Metadata) (manual.InteractionStorageEntry, error) { - executionCommands, ok := manualController.InteractionStorage[commandMetadata.ExecutionId.String()] - if !ok { - err := fmt.Sprintf("no pending commands found for execution %s", commandMetadata.ExecutionId.String()) - return manual.InteractionStorageEntry{}, manual.ErrorPendingCommandNotFound{Err: err} - } - interaction, ok := executionCommands[commandMetadata.StepId] - if !ok { - err := fmt.Sprintf("no pending commands found for execution %s -> step %s", - commandMetadata.ExecutionId.String(), - commandMetadata.StepId, - ) - return manual.InteractionStorageEntry{}, manual.ErrorPendingCommandNotFound{Err: err} - - } - return interaction, nil -} - -func (manualController *InteractionController) removeInteractionFromPending(commandMetadata execution.Metadata) error { - _, err := manualController.getPendingInteraction(commandMetadata) - if err != nil { - return err - } - // Get map of pending manual commands associated to execution - executionCommands := manualController.InteractionStorage[commandMetadata.ExecutionId.String()] - // Delete stepID-linked pending command - delete(executionCommands, commandMetadata.StepId) - - // If no pending commands associated to the execution, delete the executions map - // This is done to keep the storage clean. - if len(executionCommands) == 0 { - delete(manualController.InteractionStorage, commandMetadata.ExecutionId.String()) - } - return nil -} - -func (manualController *InteractionController) validateMatchingOutArgs(pendingEntry manual.InteractionStorageEntry, responseOutArgs cacao.Variables) ([]string, error) { - warns := []string{} - var err error = nil - for varName, variable := range responseOutArgs { - // first check that out args provided match the variables - if _, ok := pendingEntry.CommandInfo.OutArgsVariables[varName]; !ok { - err = fmt.Errorf("provided out arg %s does not match any intended out arg", varName) - return warns, manual.ErrorNonMatchingOutArgs{Err: err.Error()} - - } - // then warn if any value outside "value" has changed - if pending, ok := pendingEntry.CommandInfo.OutArgsVariables[varName]; ok { - if variable.Constant != pending.Constant { - warns = append(warns, fmt.Sprintf("provided out arg %s has different value for 'Constant' property of intended out arg. This different value is ignored.", varName)) - } - if variable.Description != pending.Description { - warns = append(warns, fmt.Sprintf("provided out arg %s has different value for 'Description' property of intended out arg. This different value is ignored.", varName)) - } - if variable.External != pending.External { - warns = append(warns, fmt.Sprintf("provided out arg %s has different value for 'External' property of intended out arg. This different value is ignored.", varName)) - } - if variable.Type != pending.Type { - warns = append(warns, fmt.Sprintf("provided out arg %s has different value for 'Type' property of intended out arg. This different value is ignored.", varName)) - } - } - } - return warns, err -} diff --git a/pkg/core/capability/openc2/openc2.go b/pkg/core/capability/openc2/openc2.go deleted file mode 100644 index 9ace5057..00000000 --- a/pkg/core/capability/openc2/openc2.go +++ /dev/null @@ -1,63 +0,0 @@ -package openc2 - -import ( - "reflect" - - "soarca/internal/logger" - "soarca/pkg/core/capability" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" - "soarca/pkg/utils/http" -) - -type OpenC2Capability struct { - httpRequest http.IHttpRequest -} - -type Empty struct{} - -const ( - openc2ResultVariableName = "__soarca_openc2_http_result__" - openc2CapabilityName = "soarca-openc2-http" -) - -var ( - component = reflect.TypeOf(Empty{}).PkgPath() - log *logger.Log -) - -func init() { - log = logger.Logger(component, logger.Info, "", logger.Json) -} - -func New(httpRequest http.IHttpRequest) *OpenC2Capability { - return &OpenC2Capability{httpRequest: httpRequest} -} - -func (OpenC2Capability *OpenC2Capability) GetType() string { - return openc2CapabilityName -} - -func (OpenC2Capability *OpenC2Capability) Execute( - metadata execution.Metadata, - context capability.Context, -) (cacao.Variables, error) { - log.Trace(metadata.ExecutionId) - - httpOptions := http.HttpOptions{ - Command: &context.Command, - Target: &context.Target, - Auth: &context.Authentication, - } - response, err := OpenC2Capability.httpRequest.Request(httpOptions) - if err != nil { - log.Error(err) - return cacao.NewVariables(), err - } - - results := cacao.NewVariables(cacao.Variable{Type: cacao.VariableTypeString, - Name: openc2ResultVariableName, - Value: string(response)}) - log.Trace("Finished openc2 execution, will return the variables: ", results) - return results, nil -} diff --git a/pkg/core/decomposer/decomposer.go b/pkg/core/decomposer/decomposer.go deleted file mode 100644 index 11acdc7e..00000000 --- a/pkg/core/decomposer/decomposer.go +++ /dev/null @@ -1,274 +0,0 @@ -package decomposer - -import ( - "errors" - "fmt" - "reflect" - - "soarca/internal/logger" - "soarca/pkg/core/executors" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" - "soarca/pkg/reporting/cases" - "soarca/pkg/reporting/reporter" - "soarca/pkg/utils/guid" - timeUtil "soarca/pkg/utils/time" - - t "time" - - "github.com/google/uuid" -) - -type Empty struct{} - -var ( - component = reflect.TypeOf(Empty{}).PkgPath() - log *logger.Log -) - -type ExecutionDetails struct { - ExecutionId uuid.UUID - PlaybookId string - Variables cacao.Variables -} - -type IDecomposer interface { - ExecuteAsync(playbook cacao.Playbook, detailsch chan ExecutionDetails) - Execute(playbook cacao.Playbook) (*ExecutionDetails, error) -} - -func init() { - log = logger.Logger(component, logger.Info, "", logger.Json) -} - -func New(actionExecutor executors.IActionExecutor, - playbookActionExecutor executors.IPlaybookExecuter, - condition executors.IConditionExecuter, - guid guid.IGuid, - reporter reporter.IWorkflowReporter, - time timeUtil.ITime) *Decomposer { - - return &Decomposer{actionExecutor: actionExecutor, - playbookActionExecutor: playbookActionExecutor, - conditionExecutor: condition, - guid: guid, - reporter: reporter, - time: time, - } -} - -type Decomposer struct { - playbook cacao.Playbook - details ExecutionDetails - actionExecutor executors.IActionExecutor - playbookActionExecutor executors.IPlaybookExecuter - conditionExecutor executors.IConditionExecuter - guid guid.IGuid - reporter reporter.IWorkflowReporter - caseManager cases.ICasesManager - time timeUtil.ITime -} - -func (decomposer *Decomposer) SetCaseManager(caseManager cases.ICasesManager) { - decomposer.caseManager = caseManager -} - -// Execute a Playbook -func (decomposer *Decomposer) ExecuteAsync(playbook cacao.Playbook, detailsch chan ExecutionDetails) { - executionId := decomposer.guid.New() - log.Debugf("Starting execution %s for Playbook %s", executionId, playbook.ID) - - details := ExecutionDetails{executionId, playbook.ID, playbook.PlaybookVariables} - decomposer.details = details - - if detailsch != nil { - detailsch <- details - } - - _ = decomposer.execute(playbook) - -} - -func (decomposer *Decomposer) Execute(playbook cacao.Playbook) (*ExecutionDetails, error) { - - executionId := decomposer.guid.New() - log.Debugf("Starting execution %s for Playbook %s", executionId, playbook.ID) - decomposer.details = ExecutionDetails{executionId, playbook.ID, playbook.PlaybookVariables} - - err := decomposer.execute(playbook) - - return &decomposer.details, err - -} - -func (decomposer *Decomposer) execute(playbook cacao.Playbook) error { - - decomposer.playbook = playbook - - stepId := playbook.WorkflowStart - - // Start case correlation and get case ID to be used in playbook - if decomposer.caseManager != nil { - startMetadata := execution.Metadata{ExecutionId: decomposer.details.ExecutionId, - PlaybookId: decomposer.playbook.ID, - StepId: stepId} - - caseIdVar := decomposer.caseManager.AddToExistingOrCreateNew(startMetadata, playbook) - playbook.PlaybookVariables.InsertOrReplace(caseIdVar) - log.Info("case id is set to: ", caseIdVar.Value) - } - - variables := cacao.NewVariables() - variables.Merge(playbook.PlaybookVariables) - - // Reporting workflow instantiation - decomposer.reporter.ReportWorkflowStart(decomposer.details.ExecutionId, playbook, decomposer.time.Now()) - - outputVariables, err := decomposer.ExecuteBranch(stepId, variables) - - decomposer.details.Variables = outputVariables - // Reporting workflow end - decomposer.reporter.ReportWorkflowEnd(decomposer.details.ExecutionId, playbook, err, decomposer.time.Now()) - - return err -} - -// Execute a Workflow branch of a Playbook -// -// Runs until it find an End step or returns an error in case there are no valid next step. -func (decomposer *Decomposer) ExecuteBranch(stepId string, scopeVariables cacao.Variables) (cacao.Variables, error) { - playbook := decomposer.playbook - log.Debug("Executing branch starting from ", stepId) - - returnVariables := cacao.NewVariables() - - for { - currentStep, ok := playbook.Workflow[stepId] - if !ok { - return cacao.NewVariables(), fmt.Errorf("step with id %s not found", stepId) - } - - log.Debug("Executing step ", stepId) - - if currentStep.Type == "end" { - break - } - - // Note: likely (but not certainly) on_success and on_faliure will be reworked - // to become workflow branching properties, with the addition of a success_condition - // boolean evaluation at step level. - // Effectively, we should thus only check for existance of on_completion, and - // report execution errors as such, not as playbook step failures - which will be handled - // with upcoming said on_success, on_failure, and success_condition properties - onCompletionStepId := currentStep.OnCompletion - if onCompletionStepId == "" { - onCompletionStepId = currentStep.OnSuccess - } - if onCompletionStepId == "" { - onCompletionStepId = currentStep.OnFailure - } - if _, ok := playbook.Workflow[onCompletionStepId]; !ok { - return cacao.NewVariables(), errors.New("empty completion step") - } - - outputVariables, err := decomposer.ExecuteStep(currentStep, scopeVariables) - - if err == nil { - stepId = onCompletionStepId - returnVariables.Merge(outputVariables) - scopeVariables.Merge(outputVariables) - } else { - return cacao.NewVariables(), fmt.Errorf("playbook execution failed at step [ %s ]. See step log for error information", stepId) - } - } - - return returnVariables, nil -} - -// Execute a single Step within a Workflow -func (decomposer *Decomposer) ExecuteStep(step cacao.Step, scopeVariables cacao.Variables) (cacao.Variables, error) { - log.Debug("Executing step type ", step.Type) - - log.Trace("Delay is set to: ", step.Delay) - decomposer.time.Sleep(t.Duration(step.Delay) * t.Millisecond) - - // Combine parent scope and Step variables - variables := cacao.NewVariables() - variables.Merge(scopeVariables) - variables.Merge(step.StepVariables) - - metadata := execution.Metadata{ - ExecutionId: decomposer.details.ExecutionId, - PlaybookId: decomposer.details.PlaybookId, - StepId: step.ID, - } - - switch step.Type { - case cacao.StepTypeAction: - actionMetadata := executors.PlaybookStepMetadata{ - Step: step, - Targets: decomposer.playbook.TargetDefinitions, - Auth: decomposer.playbook.AuthenticationInfoDefinitions, - Agent: decomposer.playbook.AgentDefinitions[step.Agent], - Variables: variables, - } - return decomposer.actionExecutor.Execute(metadata, actionMetadata) - case cacao.StepTypePlaybookAction: - return decomposer.playbookActionExecutor.Execute(metadata, step, variables) - case cacao.StepTypeIfCondition: - return decomposer.executeIfCondition(step, variables) - case cacao.StepTypeWhileCondition: - return decomposer.executeLoop(step, variables) - default: - // NOTE: This currently silently handles unknown step types. Should we return an error instead? - return cacao.NewVariables(), nil //errors.ErrUnsupported - } -} - -func (decomposer *Decomposer) executeIfCondition(step cacao.Step, - variables cacao.Variables) (cacao.Variables, error) { - metadata := execution.Metadata{ - ExecutionId: decomposer.details.ExecutionId, - PlaybookId: decomposer.details.PlaybookId, - StepId: step.ID, - } - stepId, branch, err := decomposer.conditionExecutor.Execute(metadata, - executors.Context{Step: step, Variables: variables}) - if err != nil { - return cacao.NewVariables(), err - } - if branch { - return decomposer.ExecuteBranch(stepId, variables) - } - return variables, nil -} - -func (decomposer *Decomposer) executeLoop(step cacao.Step, - variables cacao.Variables) (cacao.Variables, error) { - metadata := execution.Metadata{ - ExecutionId: decomposer.details.ExecutionId, - PlaybookId: decomposer.details.PlaybookId, - StepId: step.ID, - } - - loop := true - - for loop { - stepId, branch, err := decomposer.conditionExecutor.Execute(metadata, - executors.Context{Step: step, Variables: variables}) - if err != nil { - return cacao.NewVariables(), err - } - loop = branch - - if loop { - branchVariables, err := decomposer.ExecuteBranch(stepId, variables) - if err != nil { - return variables, err - } - variables.Merge(branchVariables) - } - - } - return variables, nil -} diff --git a/pkg/core/executors/action/action.go b/pkg/core/executors/action/action.go deleted file mode 100644 index f967f82c..00000000 --- a/pkg/core/executors/action/action.go +++ /dev/null @@ -1,200 +0,0 @@ -package action - -import ( - "errors" - "fmt" - "reflect" - "soarca/internal/logger" - "soarca/pkg/core/capability" - "soarca/pkg/core/executors" - "soarca/pkg/extensions/soarca/assignment" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" - assignmentModel "soarca/pkg/models/extensions/soarca/assignment" - "soarca/pkg/reporting/reporter" - timeUtil "soarca/pkg/utils/time" -) - -var component = reflect.TypeOf(Executor{}).PkgPath() -var log *logger.Log - -func init() { - log = logger.Logger(component, logger.Info, "", logger.Json) -} - -func New(capabilities map[string]capability.ICapability, reporter reporter.IStepReporter, time timeUtil.ITime, assigner assignment.IAssignmentExtension) *Executor { - var instance = Executor{} - instance.capabilities = capabilities - instance.reporter = reporter - instance.time = time - instance.assigner = assigner - return &instance -} - -type IExecuter interface { - Execute(metadata execution.Metadata, - step executors.PlaybookStepMetadata) (cacao.Variables, error) -} - -type Executor struct { - capabilities map[string]capability.ICapability - reporter reporter.IStepReporter - time timeUtil.ITime - assigner assignment.IAssignmentExtension -} - -type data struct { - command cacao.Command - authentication cacao.AuthenticationInformation - target cacao.AgentTarget - variables cacao.Variables - agent cacao.AgentTarget - step cacao.Step -} - -func (executor *Executor) Execute(meta execution.Metadata, - metadata executors.PlaybookStepMetadata) (cacao.Variables, error) { - - executor.reporter.ReportStepStart(meta.ExecutionId, metadata.Step, metadata.Variables, executor.time.Now()) - - returnVariables := cacao.NewVariables() - var err error - defer func() { - executor.reporter.ReportStepEnd(meta.ExecutionId, metadata.Step, returnVariables, err, executor.time.Now()) - }() - - if metadata.Step.Type != cacao.StepTypeAction { - err = errors.New("the provided step type is not compatible with this executor") - log.Error(err) - return cacao.NewVariables(), err - } - - returnVariables, err = executor.executeCommandFromArray(meta, metadata) - return returnVariables, err -} - -func (executor *Executor) executeCommandFromArray(meta execution.Metadata, - metadata executors.PlaybookStepMetadata) (cacao.Variables, error) { - returnVariables := cacao.NewVariables() - for _, command := range metadata.Step.Commands { - // NOTE: This assumes we want to run Command for every Target individually. - // Is that something we want to enforce or leave up to the capability? - for _, element := range metadata.Step.Targets { - // NOTE: What about Agent authentication? - target := metadata.Targets[element] - auth := metadata.Auth[target.AuthInfoIdentifier] - - data := data{ - command: command, - authentication: auth, - target: target, - variables: metadata.Variables, - agent: metadata.Agent, - step: metadata.Step, - } - - outputVariables, err := executor.executeCommands( - meta, - data) - - if err != nil { - log.Error("Error executing Command ", err) - return cacao.NewVariables(), err - } - log.Trace("Command executed") - - // Map defined step results into variables as described by any - // soarca-assignment step extensions. - assignedVariables := executor.evaluateAssignments(metadata.Step.StepExtensions, outputVariables) - outputVariables.Merge(assignedVariables) - - if len(metadata.Step.OutArgs) > 0 { - // If OutArgs is set, only update execution args that are explicitly referenced - outputVariables = outputVariables.Select(metadata.Step.OutArgs) - } - - returnVariables.Merge(outputVariables) - } - } - return returnVariables, nil -} - -func (executor *Executor) evaluateAssignments(extensions cacao.Extensions, results cacao.Variables) cacao.Variables { - assigned := cacao.NewVariables() - for id, raw := range extensions { - switch raw.(type) { - case assignmentModel.Assignment: - - } - model, ok := assignmentModel.DecodeAssignment(raw) - if !ok { - continue - } - log.Trace("evaluating assignment extension ", id) - produced := executor.assigner.AssignAndEvaluate(assignment.Context{ - AssignmentModel: model, - Source: results, - }) - assigned.Merge(produced) - } - return assigned -} - -func interpolateCommand(command cacao.Command, variables cacao.Variables) cacao.Command { - command.Command = variables.Interpolate(command.Command) - command.Content = variables.Interpolate(command.Content) - command.ContentB64 = variables.Interpolate(command.ContentB64) - for key, headers := range command.Headers { - var slice []string - for _, header := range headers { - slice = append(slice, variables.Interpolate(header)) - } - command.Headers[key] = slice - } - return command -} - -func interpolatedTarget(target cacao.AgentTarget, variables cacao.Variables) cacao.AgentTarget { - for key, addresses := range target.Address { - var slice []string - for _, address := range addresses { - slice = append(slice, variables.Interpolate(address)) - } - target.Address[key] = slice - } - return target -} - -func interpolateAuthentication(authentication cacao.AuthenticationInformation, variables cacao.Variables) cacao.AuthenticationInformation { - authentication.Username = variables.Interpolate(authentication.Username) - authentication.Password = variables.Interpolate(authentication.Password) - authentication.UserId = variables.Interpolate(authentication.UserId) - authentication.Token = variables.Interpolate(authentication.Token) - authentication.OauthHeader = variables.Interpolate(authentication.OauthHeader) - authentication.PrivateKey = variables.Interpolate(authentication.PrivateKey) - - return authentication - -} - -func (executor *Executor) executeCommands(metadata execution.Metadata, - data data) (cacao.Variables, error) { - - context := capability.Context{} - - if capability, ok := executor.capabilities[data.agent.Name]; ok { - context.Command = interpolateCommand(data.command, data.variables) - context.Target = interpolatedTarget(data.target, data.variables) - context.Authentication = interpolateAuthentication(data.authentication, data.variables) - context.Variables = data.variables - context.Step = data.step - returnVariables, err := capability.Execute(metadata, context) - return returnVariables, err - } else { - empty := cacao.NewVariables() - err := errors.New(fmt.Sprint("capability: ", data.agent.Name, " is not available in soarca")) - log.Error(err) - return empty, err - } - -} diff --git a/pkg/core/executors/playbook_action/playbook_action.go b/pkg/core/executors/playbook_action/playbook_action.go deleted file mode 100644 index ec1654cd..00000000 --- a/pkg/core/executors/playbook_action/playbook_action.go +++ /dev/null @@ -1,75 +0,0 @@ -package playbook_action - -import ( - "errors" - "fmt" - "reflect" - "soarca/internal/controller/database" - "soarca/internal/controller/decomposer_controller" - "soarca/internal/logger" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" - "soarca/pkg/reporting/reporter" - timeUtil "soarca/pkg/utils/time" -) - -type PlaybookAction struct { - decomposerController decomposer_controller.IController - databaseController database.IController - reporter reporter.IStepReporter - time timeUtil.ITime -} - -var component = reflect.TypeOf(PlaybookAction{}).PkgPath() -var log *logger.Log - -func init() { - log = logger.Logger(component, logger.Info, "", logger.Json) -} - -func New(controller decomposer_controller.IController, - database database.IController, reporter reporter.IStepReporter, time timeUtil.ITime) *PlaybookAction { - return &PlaybookAction{decomposerController: controller, databaseController: database, reporter: reporter, time: time} -} - -func (playbookAction *PlaybookAction) Execute(metadata execution.Metadata, - step cacao.Step, - variables cacao.Variables) (cacao.Variables, error) { - log.Trace(metadata.ExecutionId) - - playbookAction.reporter.ReportStepStart(metadata.ExecutionId, step, variables, playbookAction.time.Now()) - - var reportVars = cacao.NewVariables() - var err error - defer func() { - playbookAction.reporter.ReportStepEnd(metadata.ExecutionId, step, reportVars, err, playbookAction.time.Now()) - }() - - if step.Type != cacao.StepTypePlaybookAction { - err := errors.New(fmt.Sprint("step type is not of type ", cacao.StepTypePlaybookAction)) - log.Error(err) - return cacao.NewVariables(), err - } - - playbookRepo := playbookAction.databaseController.GetDatabaseInstance() - decomposer := playbookAction.decomposerController.NewDecomposer() - - playbook, err := playbookRepo.Read(step.PlaybookID) - if err != nil { - log.Error("failed loading the playbook from the repository in playbook action") - return cacao.NewVariables(), err - } - - playbook.PlaybookVariables.Merge(variables) - - details, err := decomposer.Execute(playbook) - if err != nil { - err = errors.New(fmt.Sprint("execution of playbook failed with error: ", err)) - log.Error(err) - reportVars = details.Variables // make sure vars are reported - return cacao.NewVariables(), err - } - reportVars = details.Variables // make sure vars are reported - return details.Variables, nil - -} diff --git a/pkg/extensions/soarca/assignment/assignment_test.go b/pkg/extensions/soarca/assignment/assignment_test.go index 3a007d6d..dd762c66 100644 --- a/pkg/extensions/soarca/assignment/assignment_test.go +++ b/pkg/extensions/soarca/assignment/assignment_test.go @@ -3,8 +3,7 @@ package assignment import ( "testing" - "soarca/pkg/models/cacao" - assignmentModel "soarca/pkg/models/extensions/soarca/assignment" + "soarca/pkg/cacao" "github.com/go-playground/assert/v2" ) @@ -22,7 +21,7 @@ func source(body string) cacao.Variables { func TestAssignPassthrough(t *testing.T) { body := `{"status": 200, "body": "ok"}` out := New().AssignAndEvaluate(Context{ - AssignmentModel: assignmentModel.Assignment{ + AssignmentModel: Assignment{ Type: "soarca-assignment", StepResult: httpResultName, Variable: "__raw_result__", @@ -40,11 +39,11 @@ func TestAssignPassthrough(t *testing.T) { func TestAssignWithJqExpression(t *testing.T) { body := `{"status": 200, "headers": {"location": "https://example.test/here"}}` out := New().AssignAndEvaluate(Context{ - AssignmentModel: assignmentModel.Assignment{ + AssignmentModel: Assignment{ Type: "soarca-assignment", StepResult: httpResultName, Variable: "__location__", - Expression: assignmentModel.Expression{Type: "jq", Expression: ".headers.location"}, + Expression: Expression{Type: "jq", Expression: ".headers.location"}, }, Source: source(body), }) @@ -56,7 +55,7 @@ func TestAssignWithJqExpression(t *testing.T) { func TestAssignMissingStepResult(t *testing.T) { out := New().AssignAndEvaluate(Context{ - AssignmentModel: assignmentModel.Assignment{StepResult: "__does_not_exist__", Variable: "__out__"}, + AssignmentModel: Assignment{StepResult: "__does_not_exist__", Variable: "__out__"}, Source: source(`{}`), }) assert.Equal(t, len(out), 0) @@ -64,10 +63,10 @@ func TestAssignMissingStepResult(t *testing.T) { func TestAssignUnknownEngine(t *testing.T) { out := New().AssignAndEvaluate(Context{ - AssignmentModel: assignmentModel.Assignment{ + AssignmentModel: Assignment{ StepResult: httpResultName, Variable: "__out__", - Expression: assignmentModel.Expression{Type: "sed", Expression: "s/a/b/"}, + Expression: Expression{Type: "sed", Expression: "s/a/b/"}, }, Source: source(`{}`), }) @@ -76,10 +75,10 @@ func TestAssignUnknownEngine(t *testing.T) { func TestAssignJqError(t *testing.T) { out := New().AssignAndEvaluate(Context{ - AssignmentModel: assignmentModel.Assignment{ + AssignmentModel: Assignment{ StepResult: httpResultName, Variable: "__out__", - Expression: assignmentModel.Expression{Type: "jq", Expression: ".headers["}, + Expression: Expression{Type: "jq", Expression: ".headers["}, }, Source: source(`{"headers": {}}`), }) @@ -88,7 +87,7 @@ func TestAssignJqError(t *testing.T) { func TestAssignMissingVariableName(t *testing.T) { out := New().AssignAndEvaluate(Context{ - AssignmentModel: assignmentModel.Assignment{StepResult: httpResultName, Variable: ""}, + AssignmentModel: Assignment{StepResult: httpResultName, Variable: ""}, Source: source(`{}`), }) assert.Equal(t, len(out), 0) diff --git a/pkg/extensions/soarca/assignment/expression/expression.go b/pkg/extensions/soarca/assignment/expression/evaluator.go similarity index 100% rename from pkg/extensions/soarca/assignment/expression/expression.go rename to pkg/extensions/soarca/assignment/expression/evaluator.go diff --git a/pkg/extensions/soarca/assignment/expression/jq/jq.go b/pkg/extensions/soarca/assignment/expression/jq/evaluator.go similarity index 100% rename from pkg/extensions/soarca/assignment/expression/jq/jq.go rename to pkg/extensions/soarca/assignment/expression/jq/evaluator.go diff --git a/pkg/extensions/soarca/assignment/expression/regex/regex.go b/pkg/extensions/soarca/assignment/expression/regex/evaluator.go similarity index 100% rename from pkg/extensions/soarca/assignment/expression/regex/regex.go rename to pkg/extensions/soarca/assignment/expression/regex/evaluator.go diff --git a/pkg/extensions/soarca/assignment/assignment.go b/pkg/extensions/soarca/assignment/extension.go similarity index 86% rename from pkg/extensions/soarca/assignment/assignment.go rename to pkg/extensions/soarca/assignment/extension.go index 56e88c1c..3015dac3 100644 --- a/pkg/extensions/soarca/assignment/assignment.go +++ b/pkg/extensions/soarca/assignment/extension.go @@ -3,11 +3,10 @@ package assignment import ( "reflect" "soarca/internal/logger" + "soarca/pkg/cacao" "soarca/pkg/extensions/soarca/assignment/expression" "soarca/pkg/extensions/soarca/assignment/expression/jq" "soarca/pkg/extensions/soarca/assignment/expression/regex" - "soarca/pkg/models/cacao" - assignmentModel "soarca/pkg/models/extensions/soarca/assignment" ) type Empty struct{} @@ -26,21 +25,22 @@ type IAssignmentExtension interface { } type Context struct { - AssignmentModel assignmentModel.Assignment + AssignmentModel Assignment Source cacao.Variables } -type Assignment struct { +// Evaluator applies assignment extension definitions to step results. +type Evaluator struct { engines map[string]expression.IExpression } -// New returns an Assignment with the default set of expression engines +// New returns an Evaluator with the default set of expression engines // registered, keyed on the engine name as it appears in an expression's // "type" field. -func New() *Assignment { +func New() *Evaluator { jqEngine := jq.New() regexEngine := regex.New() - return &Assignment{ + return &Evaluator{ engines: map[string]expression.IExpression{ jqEngine.GetEngineName(): jqEngine, regexEngine.GetEngineName(): regexEngine, @@ -53,7 +53,7 @@ func New() *Assignment { // through the matching engine. The returned Variables contain the target // variable on success, or are empty when the assignment cannot be applied // (the reason is logged). -func (assignment *Assignment) AssignAndEvaluate(context Context) cacao.Variables { +func (assignment *Evaluator) AssignAndEvaluate(context Context) cacao.Variables { variables := cacao.NewVariables() model := context.AssignmentModel diff --git a/pkg/models/extensions/soarca/assignment/assignment.go b/pkg/extensions/soarca/assignment/model.go similarity index 100% rename from pkg/models/extensions/soarca/assignment/assignment.go rename to pkg/extensions/soarca/assignment/model.go diff --git a/pkg/fins/protocol/fin_test.go b/pkg/fins/protocol/fin_test.go new file mode 100644 index 00000000..a51b7366 --- /dev/null +++ b/pkg/fins/protocol/fin_test.go @@ -0,0 +1,215 @@ +package fin + +import ( + "encoding/json" + "testing" + + "soarca/internal/workflow/capability" + "soarca/pkg/cacao" + + "github.com/go-playground/assert/v2" + "github.com/google/uuid" +) + +// These tests pin the Fin protocol JSON wire shape. + +func TestRegisterRequestJSONShape(t *testing.T) { + request := RegisterRequest{ + RegistrationToken: "shared secret, see below", + DisplayName: "Example Pong Fin", + ProtocolVersion: "1.0.0", + Capabilities: []Capability{ + { + Type: "pong", + Description: "Ping/Pong capability", + Version: "0.1.0", + StepExamples: []cacao.Step{ + { + Type: "action", + Name: "pong", + Commands: []cacao.Command{ + {Type: "manual", Command: "pong"}, + }, + }, + }, + }, + {Type: "ping", Description: "Ping capability (same Fin process, second capability)", Version: "0.1.0"}, + }, + } + + bytes, err := json.Marshal(request) + if err != nil { + t.Fatal(err) + } + + var roundTripped map[string]any + if err := json.Unmarshal(bytes, &roundTripped); err != nil { + t.Fatal(err) + } + + assert.Equal(t, roundTripped["registration_token"], "shared secret, see below") + assert.Equal(t, roundTripped["display_name"], "Example Pong Fin") + assert.Equal(t, roundTripped["protocol_version"], "1.0.0") + capabilities, ok := roundTripped["capabilities"].([]any) + if !ok { + t.Fatal("expected capabilities to be a JSON array") + } + assert.Equal(t, len(capabilities), 2) + firstCapability, ok := capabilities[0].(map[string]any) + if !ok { + t.Fatal("expected first capability to be a JSON object") + } + assert.Equal(t, firstCapability["type"], "pong") + + // Round-trip back into the Go type and confirm equality. + var decoded RegisterRequest + if err := json.Unmarshal(bytes, &decoded); err != nil { + t.Fatal(err) + } + assert.Equal(t, decoded, request) +} + +func TestRegisterResponseJSONShape(t *testing.T) { + response := RegisterResponse{ + FinId: "fin-9f2c1e3a", + FinToken: "opaque-per-fin-secret", + PollIntervalSeconds: 5, + LongPollTimeoutSeconds: 25, + JobLeaseSeconds: 60, + } + + bytes, err := json.Marshal(response) + if err != nil { + t.Fatal(err) + } + + var roundTripped map[string]any + if err := json.Unmarshal(bytes, &roundTripped); err != nil { + t.Fatal(err) + } + + assert.Equal(t, roundTripped["fin_id"], "fin-9f2c1e3a") + assert.Equal(t, roundTripped["fin_token"], "opaque-per-fin-secret") + assert.Equal(t, roundTripped["poll_interval_seconds"], float64(5)) + assert.Equal(t, roundTripped["long_poll_timeout_seconds"], float64(25)) + assert.Equal(t, roundTripped["job_lease_seconds"], float64(60)) +} + +func TestPollResponseJobJSONShape(t *testing.T) { + jobId := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + runId := uuid.MustParse("d09351a2-a075-40c8-8054-0b7c423db83f") + stepRunId := uuid.MustParse("81eff59f-d084-4324-9e0a-59e353dbd28f") + + job := Job{ + JobId: jobId, + RunId: runId, + PlaybookId: "playbook--uuid", + StepId: "action--uuid", + StepRunId: stepRunId, + CapabilityType: "ssh-executor", + LeaseExpiresInSeconds: 60, + Step: StepInfo{ + Name: "string", + Description: "string", + Timeout: 30, + Delay: 0, + }, + Commands: []Command{ + {Type: "bash", Command: "string"}, + }, + Targets: []capability.ResolvedTarget{ + { + Target: cacao.AgentTarget{ + Type: "net-address", + Name: "target", + Address: cacao.Addresses{"ipv4": {"10.0.0.1"}}, + Port: "22", + }, + Authentication: cacao.AuthenticationInformation{ + Type: "user-auth", + Username: "operator", + }, + }, + }, + Variables: cacao.NewVariables(), + } + + response := PollResponse{Job: job} + + bytes, err := json.Marshal(response) + if err != nil { + t.Fatal(err) + } + + var roundTripped map[string]any + if err := json.Unmarshal(bytes, &roundTripped); err != nil { + t.Fatal(err) + } + + jobField, ok := roundTripped["job"].(map[string]any) + if !ok { + t.Fatal("expected top-level job field") + } + assert.Equal(t, jobField["job_id"], jobId.String()) + assert.Equal(t, jobField["capability_type"], "ssh-executor") + assert.Equal(t, jobField["lease_expires_in_seconds"], float64(60)) + + var decoded PollResponse + if err := json.Unmarshal(bytes, &decoded); err != nil { + t.Fatal(err) + } + assert.Equal(t, decoded.Job.JobId, jobId) + assert.Equal(t, decoded.Job.CapabilityType, "ssh-executor") + assert.Equal(t, len(decoded.Job.Commands), 1) + assert.Equal(t, len(decoded.Job.Targets), 1) +} + +func TestResultRequestJSONShape(t *testing.T) { + targetIndex := 0 + failedCommandIndex := 1 + + request := ResultRequest{ + JobResult: JobResult{ + State: JobStateFailure, + Error: "something failed", + TargetResults: []TargetResult{ + { + TargetIndex: &targetIndex, + State: JobStateFailure, + FailedCommandIndex: &failedCommandIndex, + Error: "command 1 failed", + }, + }, + }, + } + + bytes, err := json.Marshal(request) + if err != nil { + t.Fatal(err) + } + + var decoded ResultRequest + if err := json.Unmarshal(bytes, &decoded); err != nil { + t.Fatal(err) + } + assert.Equal(t, decoded.State, JobStateFailure) + assert.Equal(t, decoded.Error, "something failed") + assert.Equal(t, len(decoded.TargetResults), 1) + assert.Equal(t, *decoded.TargetResults[0].TargetIndex, 0) + assert.Equal(t, *decoded.TargetResults[0].FailedCommandIndex, 1) +} + +func TestStatusPingResponseJSONShape(t *testing.T) { + response := StatusPingResponse{Action: "cancel"} + + bytes, err := json.Marshal(response) + if err != nil { + t.Fatal(err) + } + + var roundTripped map[string]any + if err := json.Unmarshal(bytes, &roundTripped); err != nil { + t.Fatal(err) + } + assert.Equal(t, roundTripped["action"], "cancel") +} diff --git a/pkg/fins/protocol/protocol.go b/pkg/fins/protocol/protocol.go new file mode 100644 index 00000000..cea1efdb --- /dev/null +++ b/pkg/fins/protocol/protocol.go @@ -0,0 +1,54 @@ +package fin + +// RegisterRequest is the body of POST /fin/register. +type RegisterRequest struct { + // RegistrationToken must match FIN_REGISTRATION_TOKEN. + RegistrationToken string `json:"registration_token" validate:"required"` + DisplayName string `json:"display_name,omitempty"` + ProtocolVersion string `json:"protocol_version,omitempty"` + Capabilities []Capability `json:"capabilities" validate:"required"` +} + +// RegisterResponse is returned on successful registration. +type RegisterResponse struct { + FinId string `json:"fin_id"` + FinToken string `json:"fin_token"` + // Server-provided polling and lease defaults. + PollIntervalSeconds int `json:"poll_interval_seconds"` + LongPollTimeoutSeconds int `json:"long_poll_timeout_seconds"` + JobLeaseSeconds int `json:"job_lease_seconds"` +} + +// PollRequest is the body of POST /fin/poll. The calling Fin is +// identified and authenticated by its Authorization: Bearer fin_token +// header — fin_id and capability types are never resent here; SOARCA +// already knows both server-side, keyed off the token. +type PollRequest struct { + // Optional hint for available worker slots on the Fin side. + ConcurrencyAvailable int `json:"concurrency_available,omitempty"` +} + +// PollResponse is returned when a job is available. +type PollResponse struct { + Job Job `json:"job"` +} + +// ResultRequest is the body of PUT /fin/jobs/{job_id}. +type ResultRequest struct { + JobResult +} + +// StatusPingRequest extends an active lease for long-running jobs. +type StatusPingRequest struct { + Progress string `json:"progress,omitempty"` +} + +// StatusPingResponse carries optional instructions (for example, cancel). +type StatusPingResponse struct { + Action string `json:"action,omitempty"` +} + +// ListResponse is returned by GET /fin/. +type ListResponse struct { + Fins []Record `json:"fins"` +} diff --git a/pkg/fins/protocol/records.go b/pkg/fins/protocol/records.go new file mode 100644 index 00000000..281a6089 --- /dev/null +++ b/pkg/fins/protocol/records.go @@ -0,0 +1,223 @@ +// Package fin defines data models for the Fin HTTP/JSON protocol. +package fin + +import ( + "fmt" + "strings" + "time" + + "soarca/internal/workflow/capability" + "soarca/pkg/cacao" + + "github.com/google/uuid" +) + +// Capability describes one capability a Fin can execute. +type Capability struct { + // Type is the routing key used to match jobs to Fins. + Type string `bson:"type" json:"type" validate:"required"` + // Description is free text for humans (dashboards, logs) — it plays no + // role in routing. + Description string `bson:"description,omitempty" json:"description,omitempty"` + // Version is the capability implementation's own version string, for + // operator/debugging visibility only. + Version string `bson:"version,omitempty" json:"version,omitempty"` + // StepExamples are optional, illustrative CACAO action steps showing + // how a playbook author would invoke this capability (agent, commands, + // targets, ...), surfaced to help playbook authors — they are never + // interpreted or validated by SOARCA. A capability may offer more than + // one example (e.g. to illustrate different commands or target shapes). + StepExamples []cacao.Step `bson:"step_examples,omitempty" json:"step_examples,omitempty"` +} + +// Record is a persisted Fin registration with liveness metadata. +type Record struct { + // FinId is server-assigned at registration (never client-chosen), + // avoiding id collisions and any implicit trust that a Fin picks a + // unique id for itself. + FinId string `bson:"_id" json:"fin_id"` + // FinTokenHash is a one-way hash of the per-Fin credential (fin_token) + // returned once at registration. The plaintext token is never + // persisted: SOARCA hashes an incoming Authorization: Bearer token the + // same way and compares hashes, so a database read/leak alone cannot + // recover a usable credential. + FinTokenHash string `bson:"fin_token_hash" json:"-"` + // DisplayName is free text for humans/logs only; it plays no role in + // routing or identity. + DisplayName string `bson:"display_name,omitempty" json:"display_name,omitempty"` + ProtocolVersion string `bson:"protocol_version,omitempty" json:"protocol_version,omitempty"` + Capabilities []Capability `bson:"capabilities" json:"capabilities"` + RegisteredAt time.Time `bson:"registered_at" json:"registered_at"` + // LastSeen is updated by poll, result submission, and status ping. + LastSeen time.Time `bson:"last_seen" json:"last_seen"` + // Stale is computed at read time and is not persisted. + Stale bool `bson:"-" json:"stale"` +} + +// JobState is the aggregated outcome of a job. +type JobState string + +const ( + JobStateSuccess JobState = "success" + JobStateFailure JobState = "failure" +) + +// Job is one pollable unit of work for a Fin. +type Job struct { + // JobId identifies this specific poll-able unit of work (== the lease + // handle used for result submission and status pings). + JobId uuid.UUID `bson:"_id" json:"job_id"` + // StepRunId disambiguates repeated invocations of the same StepId. + RunId uuid.UUID `bson:"run_id" json:"run_id"` + PlaybookId string `bson:"playbook_id" json:"playbook_id"` + StepId string `bson:"step_id" json:"step_id"` + StepRunId uuid.UUID `bson:"step_run_id" json:"step_run_id"` + // CapabilityType is the routing key this job was queued under — it + // matches Capability.Type of whichever Fin ultimately claims it. + CapabilityType string `bson:"capability_type" json:"capability_type"` + // Lease timeout; jobs may be requeued after expiry. + LeaseExpiresInSeconds int `bson:"lease_expires_in_seconds" json:"lease_expires_in_seconds"` + Step StepInfo `bson:"step" json:"step"` + Commands []Command `bson:"commands" json:"commands"` + Targets []capability.ResolvedTarget `bson:"targets" json:"targets"` + Variables cacao.Variables `bson:"variables" json:"variables"` +} + +// StepInfo carries the metadata a Fin needs. +type StepInfo struct { + Name string `bson:"name,omitempty" json:"name,omitempty"` + Description string `bson:"description,omitempty" json:"description,omitempty"` + Timeout int `bson:"timeout,omitempty" json:"timeout,omitempty"` + Delay int `bson:"delay,omitempty" json:"delay,omitempty"` +} + +// Command is one command in a job. +type Command struct { + Type string `bson:"type" json:"type"` + Command string `bson:"command,omitempty" json:"command,omitempty"` + CommandB64 string `bson:"command_b64,omitempty" json:"command_b64,omitempty"` + Content string `bson:"content,omitempty" json:"content,omitempty"` + ContentB64 string `bson:"content_b64,omitempty" json:"content_b64,omitempty"` + Headers map[string][]string `bson:"headers,omitempty" json:"headers,omitempty"` +} + +// TargetResult is optional, additive per-target diagnostic detail on a job +// result — useful for the reporter/GUI/audit log, but never consulted by +// playbook control flow, which only ever sees the single aggregated +// (State, Variables) pair on JobResult. +type TargetResult struct { + // TargetIndex identifies which entry in the job's Targets this result + // is for. Nil when the job had no targets at all. + TargetIndex *int `bson:"target_index,omitempty" json:"target_index,omitempty"` + State JobState `bson:"state" json:"state"` + // FailedCommandIndex identifies which command in Commands aborted this + // target's sequence, if any. + FailedCommandIndex *int `bson:"failed_command_index,omitempty" json:"failed_command_index,omitempty"` + Variables cacao.Variables `bson:"variables,omitempty" json:"variables,omitempty"` + Error string `bson:"error,omitempty" json:"error,omitempty"` +} + +// JobResult is submitted by a Fin after job completion. +type JobResult struct { + State JobState `bson:"state" json:"state" validate:"required"` + Variables cacao.Variables `bson:"variables,omitempty" json:"variables,omitempty"` + Error string `bson:"error,omitempty" json:"error,omitempty"` + TargetResults []TargetResult `bson:"target_results,omitempty" json:"target_results,omitempty"` +} + +// Errors ###################################################################### + +// ErrRegistrationTokenInvalid is returned for invalid registration tokens. +type ErrRegistrationTokenInvalid struct{} + +func (e ErrRegistrationTokenInvalid) Error() string { + return "invalid or missing registration token" +} + +// ErrFinTokenInvalid is returned for unknown Fin tokens. +type ErrFinTokenInvalid struct{} + +func (e ErrFinTokenInvalid) Error() string { + return "invalid or unknown fin token" +} + +// ErrFinNotFound indicates no registration exists for the FinId. +type ErrFinNotFound struct { + FinId string +} + +func (e ErrFinNotFound) Error() string { + return "no fin registered with id " + e.FinId +} + +// ErrAlreadyRegistered indicates a duplicate Fin registration ID. +type ErrAlreadyRegistered struct { + FinId string +} + +func (e ErrAlreadyRegistered) Error() string { + return "a fin is already registered with id " + e.FinId +} + +// ErrJobNotFound indicates the job does not exist. +type ErrJobNotFound struct { + JobId string +} + +func (e ErrJobNotFound) Error() string { + return "no job found with id " + e.JobId +} + +// ErrJobNotLeasedToFin indicates a lease mismatch for the acting Fin. +type ErrJobNotLeasedToFin struct { + JobId string + FinId string +} + +func (e ErrJobNotLeasedToFin) Error() string { + return "job " + e.JobId + " is not leased to fin " + e.FinId +} + +// ErrNoCapableFin indicates no Fin is registered for the capability type. +type ErrNoCapableFin struct { + CapabilityType string +} + +func (e ErrNoCapableFin) Error() string { + return "no fin is registered for capability type " + e.CapabilityType +} + +// ErrOnlyStaleCapableFins indicates all capable Fins are stale. +type ErrOnlyStaleCapableFins struct { + CapabilityType string + FinIds []string + StaleAfter time.Duration +} + +func (e ErrOnlyStaleCapableFins) Error() string { + return fmt.Sprintf( + "every fin registered for capability type %s has not been seen in over %s (fin ids: %s); assuming none are still running", + e.CapabilityType, e.StaleAfter, strings.Join(e.FinIds, ", "), + ) +} + +// ErrRegistrationDisabled indicates that FIN registration is disabled. +type ErrRegistrationDisabled struct{} + +func (e ErrRegistrationDisabled) Error() string { + return "fin registration is not configured" +} + +// ErrNoCapabilities indicates no capabilities were provided during registration. +type ErrNoCapabilities struct{} + +func (e ErrNoCapabilities) Error() string { + return "at least one capability is required" +} + +// ErrCapabilityTypeEmpty indicates a capability has an empty type. +type ErrCapabilityTypeEmpty struct{} + +func (e ErrCapabilityTypeEmpty) Error() string { + return "every capability requires a non-empty type" +} diff --git a/pkg/integration/thehive/common/mappings/mappings.go b/pkg/integration/thehive/common/mappings/mappings.go deleted file mode 100644 index 91d0ef77..00000000 --- a/pkg/integration/thehive/common/mappings/mappings.go +++ /dev/null @@ -1,74 +0,0 @@ -package mappings - -import ( - "fmt" - "reflect" - "soarca/internal/logger" -) - -var ( - component = reflect.TypeOf(ExecutionCaseMap{}).PkgPath() - log *logger.Log -) - -func init() { - log = logger.Logger(component, logger.Info, "", logger.Json) -} - -// ############################### Playbook to TheHive ID mappings - -type SOARCATheHiveMap struct { - ExecutionsCaseMaps map[string]ExecutionCaseMap -} -type ExecutionCaseMap struct { - caseId string - stepsTasksMap map[string]string -} - -// TODO: Change to using observables instead of updating the tasks descriptions - -func (soarcaTheHiveMap *SOARCATheHiveMap) CheckExecutionCaseExists(executionId string) error { - if _, ok := soarcaTheHiveMap.ExecutionsCaseMaps[executionId]; !ok { - return fmt.Errorf("case not found for execution id %s", executionId) - } - return nil -} -func (soarcaTheHiveMap *SOARCATheHiveMap) CheckExecutionStepTaskExists(executionId string, stepId string) error { - if _, ok := soarcaTheHiveMap.ExecutionsCaseMaps[executionId].stepsTasksMap[stepId]; !ok { - return fmt.Errorf("task not found for execution id %s for step id %s", executionId, stepId) - } - return nil -} - -func (soarcaTheHiveMap *SOARCATheHiveMap) RegisterExecutionInCase(executionId string, caseId string) error { - soarcaTheHiveMap.ExecutionsCaseMaps[executionId] = ExecutionCaseMap{ - caseId: caseId, - stepsTasksMap: map[string]string{}, - } - log.Info(fmt.Sprintf("registering execution: %s, case id: %s", executionId, caseId)) - - return nil -} -func (soarcaTheHiveMap *SOARCATheHiveMap) RegisterStepTaskInCase(executionId string, stepId string, taskId string) { - soarcaTheHiveMap.ExecutionsCaseMaps[executionId].stepsTasksMap[stepId] = taskId -} - -func (soarcaTheHiveMap *SOARCATheHiveMap) RetrieveCaseId(executionId string) (string, error) { - err := soarcaTheHiveMap.CheckExecutionCaseExists(executionId) - if err != nil { - return "", err - } - return soarcaTheHiveMap.ExecutionsCaseMaps[executionId].caseId, nil -} - -func (soarcaTheHiveMap *SOARCATheHiveMap) RetrieveTaskId(executionId string, stepId string) (string, error) { - err := soarcaTheHiveMap.CheckExecutionCaseExists(executionId) - if err != nil { - return "", err - } - err = soarcaTheHiveMap.CheckExecutionStepTaskExists(executionId, stepId) - if err != nil { - return "", err - } - return soarcaTheHiveMap.ExecutionsCaseMaps[executionId].stepsTasksMap[stepId], nil -} diff --git a/pkg/models/api/api.go b/pkg/models/api/api.go deleted file mode 100644 index cdc6820b..00000000 --- a/pkg/models/api/api.go +++ /dev/null @@ -1,12 +0,0 @@ -package api - -import "time" - -type PlaybookMeta struct { - ID string `json:"id"` - Name string `json:"name"` - Description string `json:"description"` - ValidFrom time.Time `json:"valid_from"` - ValidUntil time.Time `json:"valid_until"` - Labels []string `json:"labels"` -} diff --git a/pkg/models/api/execution.go b/pkg/models/api/execution.go deleted file mode 100644 index bb6a730f..00000000 --- a/pkg/models/api/execution.go +++ /dev/null @@ -1,8 +0,0 @@ -package api - -import "github.com/google/uuid" - -type Execution struct { - ExecutionId uuid.UUID `json:"execution_id" validate:"required" example:"2c855cd6-bbce-402f-a143-3d6eec346c08"` - PlaybookId string `json:"payload" validate:"required" example:"playbook--0cec398c-db69-4f17-bde4-8ecbcc4a8879"` -} diff --git a/pkg/models/api/manual.go b/pkg/models/api/manual.go deleted file mode 100644 index 35b3acd4..00000000 --- a/pkg/models/api/manual.go +++ /dev/null @@ -1,31 +0,0 @@ -package api - -import ( - "soarca/pkg/models/cacao" - "soarca/pkg/models/manual" -) - -// Object interfaced to users storing info about pending manual commands -// TODO: change to manualcommandinfo -type InteractionCommandData struct { - Type string `bson:"type" json:"type" validate:"required" example:"execution-status"` // The type of this content - ExecutionId string `bson:"execution_id" json:"execution_id" validate:"required"` // The id of the execution - PlaybookId string `bson:"playbook_id" json:"playbook_id" validate:"required"` // The id of the CACAO playbook executed by the execution - StepId string `bson:"step_id" json:"step_id" validate:"required"` // The id of the step executed by the execution - Description string `bson:"description" json:"description" validate:"required"` // The description from the workflow step - Command string `bson:"command" json:"command" validate:"required"` // The command for the agent either command - CommandIsBase64 bool `bson:"commandb64,omitempty" json:"commandb64,omitempty"` // Indicates if the command is in b64 - Target cacao.AgentTarget `bson:"target" json:"target" validate:"required"` // Map of cacao agent-target with the target(s) of this command - OutVariables cacao.Variables `bson:"out_args" json:"out_args" validate:"required"` // Map of cacao variables handled in the step out args with current values and definitions -} - -// The object posted on the manual API Continue() payload -type ManualOutArgsUpdatePayload struct { - Type string `bson:"type" json:"type" validate:"required" example:"string"` // The type of this content - ExecutionId string `bson:"execution_id" json:"execution_id" validate:"required"` // The id of the execution - PlaybookId string `bson:"playbook_id" json:"playbook_id" validate:"required"` // The id of the CACAO playbook executed by the execution - StepId string `bson:"step_id" json:"step_id" validate:"required"` // The id of the step executed by the execution - ResponseStatus manual.ManualResponseStatus `bson:"response_status" json:"response_status" validate:"required"` // Indicates status of command - - ResponseOutArgs cacao.Variables `bson:"response_out_args" json:"response_out_args" validate:"required"` // Map of cacao variables storing the out args value, handled in the step out args, with current values and definitions -} diff --git a/pkg/models/execution/execution.go b/pkg/models/execution/execution.go deleted file mode 100644 index 6cfedaa4..00000000 --- a/pkg/models/execution/execution.go +++ /dev/null @@ -1,11 +0,0 @@ -package execution - -import ( - "github.com/google/uuid" -) - -type Metadata struct { - ExecutionId uuid.UUID - PlaybookId string - StepId string -} diff --git a/pkg/models/fin/fin.go b/pkg/models/fin/fin.go deleted file mode 100644 index 1eb7b0ae..00000000 --- a/pkg/models/fin/fin.go +++ /dev/null @@ -1,173 +0,0 @@ -package fin - -import ( - "encoding/json" - "soarca/pkg/models/cacao" - "time" -) - -// command constants -const ( - MessageTypeAck = "ack" - MessageTypeNack = "nack" - MessageTypeRegister = "register" - MessageTypeUnregister = "unregister" - MessageTypeCommand = "command" - MessageTypeResult = "result" - MessageTypePause = "pause" - MessageTypeResume = "resume" - MessageTypeStop = "stop" -) - -// Ack -type Ack struct { - Type string `json:"type"` - MessageId string `json:"message_id"` -} - -// Nack -type Nack struct { - Type string `json:"type"` - MessageId string `json:"message_id"` -} - -// Register message structure -type Register struct { - Type string `json:"type"` - MessageId string `json:"message_id"` - FinID string `json:"fin_id"` - Name string `json:"fin_name"` - ProtocolVersion string `json:"protocol_version"` - Security Security `json:"security"` - Capabilities []Capability `json:"capabilities"` - Meta Meta `json:"meta,omitempty"` -} - -// Capability register message substructure -type Capability struct { - Id string `json:"capability_id"` - Name string `json:"name"` - Version string `json:"version"` - Step map[string]Step `json:"step,omitempty"` - Agent map[string]cacao.AgentTarget `json:"agent,omitempty"` -} - -// Step structure as example to the executor -type Step struct { - Type string `json:"type"` - Name string `json:"name"` - Description string `json:"description"` - ExternalReferences []cacao.ExternalReferences `json:"external_references"` - Command string `json:"command"` - Target string `json:"target"` -} - -// Unregister command structure -type Unregister struct { - Type string `json:"type"` - MessageId string `json:"message_id"` - Id string `json:"capability_id"` - FinID string `json:"fin_id"` - All string `json:"all"` -} - -// Command -type Command struct { - Type string `json:"type"` - MessageId string `json:"message_id"` - CommandSubstructure CommandSubstructure `json:"command"` - Meta Meta `json:"meta"` -} - -// Command substructure used by the command message -type CommandSubstructure struct { - Command string `json:"command"` - Authentication cacao.AuthenticationInformation `json:"authentication"` - Context Context `json:"context"` - Variables map[string]cacao.Variable `json:"variables"` -} - -// Result message structure -type Result struct { - Type string `json:"type"` - MessageId string `json:"message_id"` - ResultStructure ResultStructure `json:"result"` - Meta Meta `json:"meta"` -} - -// Result substructure used by the result message -type ResultStructure struct { - State string `json:"state"` - Context Context `json:"context"` - Variables map[string]cacao.Variable `json:"variables"` -} - -// Control message structure -type Control struct { - Type string `json:"type"` - MessageId string `json:"message_id"` - CapabilityId string `json:"capability_id"` -} - -// Status message structure -type Status struct { - Type string `json:"type"` - MessageId string `json:"message_id"` - CapabilityId string `json:"capability_id"` - Progress string `json:"progress"` -} - -// Security message substructure -type Security struct { - Version string `json:"version"` - ChannelSecurity string `json:"channel_security"` -} - -// Context message substructure -type Context struct { - CompletedOn time.Time `json:"completed_on"` - GeneratedOn time.Time `json:"generated_on"` - Timeout int `json:"timeout"` - Delay int `json:"delay"` - StepId string `json:"step_id"` - PlaybookId string `json:"playbook_id"` - ExecutionId string `json:"execution_id"` -} - -// Meta message substructure -type Meta struct { - Timestamp time.Time `json:"timestamp"` - SenderId string `json:"sender_id"` -} - -type Message struct { - Type string `json:"type"` - MessageId string `json:"message_id"` -} - -func NewCommand() Command { - instance := Command{} - instance.Type = MessageTypeCommand - instance.CommandSubstructure.Context.Timeout = 1 - //instance.CommandSubstructure.Context.GeneratedOn = time.Now() - - return instance -} - -func NewAck(messageId string) Ack { - ack := Ack{Type: MessageTypeAck, MessageId: messageId} - return ack -} - -func NewNack(messageId string) Nack { - nack := Nack{Type: MessageTypeNack, MessageId: messageId} - return nack -} - -func Decode(data []byte, object any) error { - return json.Unmarshal(data, object) -} - -func Encode(object any) ([]byte, error) { - return json.Marshal(object) -} diff --git a/pkg/models/fin/finmodel_test.go b/pkg/models/fin/finmodel_test.go deleted file mode 100644 index fe9606e7..00000000 --- a/pkg/models/fin/finmodel_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package fin - -import ( - "testing" - "time" - - "github.com/go-playground/assert/v2" -) - -func TestFinCommandCreation(t *testing.T) { - command := NewCommand() - // Check if set - assert.Equal(t, command.Type, MessageTypeCommand) - assert.Equal(t, command.CommandSubstructure.Context.Timeout, 1) - assert.Equal(t, time.Time.IsZero(command.CommandSubstructure.Context.GeneratedOn), true) - - // Check if not set - assert.Equal(t, command.MessageId, "") - assert.Equal(t, command.Meta.SenderId, "") - assert.Equal(t, time.Time.IsZero(command.Meta.Timestamp), true) - assert.Equal(t, time.Time.IsZero(command.CommandSubstructure.Context.CompletedOn), true) - assert.Equal(t, command.CommandSubstructure.Context.Delay, 0) - assert.Equal(t, command.CommandSubstructure.Context.StepId, "") - assert.Equal(t, command.CommandSubstructure.Context.ExecutionId, "") - assert.Equal(t, command.CommandSubstructure.Context.PlaybookId, "") -} diff --git a/pkg/reporting/reporter/downstream_reporter/cache/cache.go b/pkg/reporting/reporter/downstream_reporter/cache/cache.go deleted file mode 100644 index 0dc2fa52..00000000 --- a/pkg/reporting/reporter/downstream_reporter/cache/cache.go +++ /dev/null @@ -1,295 +0,0 @@ -package cache - -import ( - b64 "encoding/base64" - "errors" - "fmt" - "slices" - "soarca/pkg/models/cacao" - cache_report "soarca/pkg/models/cache" - itime "soarca/pkg/utils/time" - "sync" - "time" - - "github.com/google/uuid" -) - -const MaxExecutions int = 10 - -type Cache struct { - Size int - timeUtil itime.ITime - Cache map[string]cache_report.ExecutionEntry // Cached up to max - fifoRegister []string // Used for O(1) FIFO cache management - mutex sync.Mutex -} - -func New(timeUtil itime.ITime, maxExecutions int) *Cache { - return &Cache{ - Size: maxExecutions, - Cache: make(map[string]cache_report.ExecutionEntry), - timeUtil: timeUtil, - mutex: sync.Mutex{}, - } -} - -// ############################### Atomic cache access operations (mutex-protection) - -func (cacheReporter *Cache) getAllExecutions() ([]cache_report.ExecutionEntry, error) { - executions := make([]cache_report.ExecutionEntry, 0) - // NOTE: fetched via fifo register key reference as is ordered array, - // this is needed to test and report back ordered executions stored - - // Lock - cacheReporter.mutex.Lock() - defer cacheReporter.mutex.Unlock() - for _, executionEntryKey := range cacheReporter.fifoRegister { - // NOTE: cached executions are passed by reference, so they must not be modified - entry, ok := cacheReporter.Cache[executionEntryKey] - if !ok { - // Unlock - return []cache_report.ExecutionEntry{}, errors.New("internal error. cache fifo register and cache executions mismatch") - } - executions = append(executions, entry) - } - - // Unlocked - return executions, nil -} - -func (cacheReporter *Cache) getExecution(executionKey uuid.UUID) (cache_report.ExecutionEntry, error) { - - executionKeyStr := executionKey.String() - // No need for mutex as is one-line access - executionEntry, ok := cacheReporter.Cache[executionKeyStr] - - if !ok { - err := errors.New("execution is not in cache. consider increasing cache size") - return cache_report.ExecutionEntry{}, err - // TODO Retrieve from database and push to cache - } - return executionEntry, nil -} - -// Adding executions in FIFO logic -func (cacheReporter *Cache) addExecutionFIFO(newExecutionEntry cache_report.ExecutionEntry) error { - - if len(cacheReporter.fifoRegister) != len(cacheReporter.Cache) { - return errors.New("cache fifo register and content are desynchronized") - } - - newExecutionEntryKey := newExecutionEntry.ExecutionId.String() - - // Lock - cacheReporter.mutex.Lock() - defer cacheReporter.mutex.Unlock() - - if _, ok := cacheReporter.Cache[newExecutionEntryKey]; ok { - return errors.New("there is already an execution in the cache with the same execution id") - } - if len(cacheReporter.fifoRegister) >= cacheReporter.Size { - - firstExecution := cacheReporter.fifoRegister[0] - cacheReporter.fifoRegister = cacheReporter.fifoRegister[1:] - delete(cacheReporter.Cache, firstExecution) - cacheReporter.fifoRegister = append(cacheReporter.fifoRegister, newExecutionEntryKey) - cacheReporter.Cache[newExecutionEntryKey] = newExecutionEntry - - return nil - // Unlocked - } - cacheReporter.fifoRegister = append(cacheReporter.fifoRegister, newExecutionEntryKey) - cacheReporter.Cache[newExecutionEntryKey] = newExecutionEntry - - return nil - // Unlocked -} - -func (cacheReporter *Cache) upateEndExecutionWorkflow(executionId uuid.UUID, workflowError error, at time.Time) error { - // The cache should stay locked for the whole modification period - // in order to prevent e.g. the execution data being popped-out due to FIFO - // while its status or some of its steps are being updated - - // Lock - cacheReporter.mutex.Lock() - defer cacheReporter.mutex.Unlock() - - executionEntry, err := cacheReporter.getExecution(executionId) - if err != nil { - return err - } - - if workflowError != nil { - executionEntry.Error = workflowError - executionEntry.Status = cache_report.Failed - } else { - executionEntry.Status = cache_report.SuccessfullyExecuted - } - executionEntry.Ended = at - cacheReporter.Cache[executionId.String()] = executionEntry - - return nil - // Unlocked -} - -func (cacheReporter *Cache) addStartExecutionStep(executionId uuid.UUID, newStepData cache_report.StepResult) error { - // Locked - cacheReporter.mutex.Lock() - defer cacheReporter.mutex.Unlock() - - executionEntry, err := cacheReporter.getExecution(executionId) - if err != nil { - return err - } - - if executionEntry.Status != cache_report.Ongoing { - return errors.New("trying to report on the execution of a step for an already reportedly terminated playbook execution") - } - _, alreadyThere := executionEntry.StepResults[newStepData.StepId] - if alreadyThere { - // TODO: must fix: all steps should start empty values but already present. Check should be - // done on Step.Started > 0 time - // - // Should divide between instanciation of step, and modification of step, - // with respective checks step status - return errors.New("a step execution start was already reported for this step. ignoring") - } - - executionEntry.StepResults[newStepData.StepId] = newStepData - // New code - cacheReporter.Cache[executionId.String()] = executionEntry - - return nil - // Unlocked -} - -func (cacheReporter *Cache) upateEndExecutionStep(executionId uuid.UUID, stepId string, returnVars cacao.Variables, stepError error, acceptedStepStati []cache_report.Status, at time.Time) error { - // Locked - cacheReporter.mutex.Lock() - defer cacheReporter.mutex.Unlock() - - executionEntry, err := cacheReporter.getExecution(executionId) - if err != nil { - return err - } - - executionStepResult, ok := executionEntry.StepResults[stepId] - if !ok { - // TODO: must fix: all steps should start empty values but already present. Check should be - // done on Step.Started > 0 time - return errors.New("trying to update a step which was not (yet?) recorded in the cache") - // Unlocked - } - - if !slices.Contains(acceptedStepStati, executionStepResult.Status) { - return fmt.Errorf("step status precondition not met for step update [step status: %s]", executionStepResult.Status.String()) - } - - if stepError != nil { - executionStepResult.Error = stepError - executionStepResult.Status = cache_report.ServerSideError - } else { - executionStepResult.Status = cache_report.SuccessfullyExecuted - } - executionStepResult.Ended = at - executionStepResult.Variables = returnVars - executionEntry.StepResults[stepId] = executionStepResult - cacheReporter.Cache[executionId.String()] = executionEntry - - return nil - // Unlocked -} - -// ############################### Informer interface - -func (cacheReporter *Cache) GetExecutions() ([]cache_report.ExecutionEntry, error) { - executions, err := cacheReporter.getAllExecutions() - return executions, err -} - -func (cacheReporter *Cache) GetExecutionReport(executionKey uuid.UUID) (cache_report.ExecutionEntry, error) { - - executionEntry, err := cacheReporter.getExecution(executionKey) - if err != nil { - return cache_report.ExecutionEntry{}, err - } - report := executionEntry - - return report, nil -} - -// ############################### Reporting interface - -func (cacheReporter *Cache) ReportWorkflowStart(executionId uuid.UUID, playbook cacao.Playbook, at time.Time) error { - - newExecutionEntry := cache_report.ExecutionEntry{ - ExecutionId: executionId, - PlaybookId: playbook.ID, - Name: playbook.Name, - Description: playbook.Description, - Started: at, - Ended: time.Time{}, - StepResults: map[string]cache_report.StepResult{}, - Status: cache_report.Ongoing, - } - err := cacheReporter.addExecutionFIFO(newExecutionEntry) - if err != nil { - return err - } - return nil -} - -func (cacheReporter *Cache) ReportWorkflowEnd(executionId uuid.UUID, playbook cacao.Playbook, workflowError error, at time.Time) error { - - err := cacheReporter.upateEndExecutionWorkflow(executionId, workflowError, at) - return err -} - -func (cacheReporter *Cache) ReportStepStart(executionId uuid.UUID, step cacao.Step, variables cacao.Variables, at time.Time) error { - - commandsB64 := []string{} - isAutomated := true - for _, cmd := range step.Commands { - if cmd.Type == cacao.CommandTypeManual { - isAutomated = false - } - if cmd.CommandB64 != "" { - commandsB64 = append(commandsB64, cmd.CommandB64) - } else { - cmdB64 := b64.StdEncoding.EncodeToString([]byte(cmd.Command)) - commandsB64 = append(commandsB64, cmdB64) - } - } - - newStep := cache_report.StepResult{ - ExecutionId: executionId, - StepId: step.ID, - Name: step.Name, - Description: step.Description, - //Started: cacheReporter.timeUtil.Now(), - Started: at, - Ended: time.Time{}, - Variables: variables, - CommandsB64: commandsB64, - Status: cache_report.Ongoing, - Error: nil, - IsAutomated: isAutomated, - } - - err := cacheReporter.addStartExecutionStep(executionId, newStep) - - return err -} - -func (cacheReporter *Cache) ReportStepEnd(executionId uuid.UUID, step cacao.Step, returnVars cacao.Variables, stepError error, at time.Time) error { - - // stepId, err := uuid.Parse(step.ID) - // if err != nil { - // return fmt.Errorf("could not parse to uuid the step id: %s", step.ID) - // } - - acceptedStepStati := []cache_report.Status{cache_report.Ongoing} - err := cacheReporter.upateEndExecutionStep(executionId, step.ID, returnVars, stepError, acceptedStepStati, at) - - return err -} diff --git a/pkg/reporting/reporter/downstream_reporter/downstream_reporter.go b/pkg/reporting/reporter/downstream_reporter/downstream_reporter.go deleted file mode 100644 index ff31693d..00000000 --- a/pkg/reporting/reporter/downstream_reporter/downstream_reporter.go +++ /dev/null @@ -1,16 +0,0 @@ -package downstream_reporter - -import ( - "soarca/pkg/models/cacao" - "time" - - "github.com/google/uuid" -) - -type IDownStreamReporter interface { - ReportWorkflowStart(executionId uuid.UUID, playbook cacao.Playbook, at time.Time) error - ReportWorkflowEnd(executionId uuid.UUID, playbook cacao.Playbook, err error, at time.Time) error - - ReportStepStart(executionId uuid.UUID, step cacao.Step, stepResults cacao.Variables, at time.Time) error - ReportStepEnd(executionId uuid.UUID, step cacao.Step, stepResults cacao.Variables, err error, at time.Time) error -} diff --git a/pkg/utils/utils.go b/pkg/utils/env.go similarity index 100% rename from pkg/utils/utils.go rename to pkg/utils/env.go diff --git a/pkg/utils/guid/guid.go b/pkg/utils/guid/generator.go similarity index 100% rename from pkg/utils/guid/guid.go rename to pkg/utils/guid/generator.go diff --git a/pkg/utils/http/http_test.go b/pkg/utils/http/http_test.go index 91413542..cddb939b 100644 --- a/pkg/utils/http/http_test.go +++ b/pkg/utils/http/http_test.go @@ -1,3 +1,7 @@ +//go:build integration + +// These tests require the httpbin service from deployments/docker/testing. + package http import ( @@ -7,7 +11,7 @@ import ( "strconv" "testing" - "soarca/pkg/models/cacao" + "soarca/pkg/cacao" "github.com/go-playground/assert/v2" ) diff --git a/pkg/utils/http/http.go b/pkg/utils/http/request.go similarity index 99% rename from pkg/utils/http/http.go rename to pkg/utils/http/request.go index 7ef20313..c2b6ec79 100644 --- a/pkg/utils/http/http.go +++ b/pkg/utils/http/request.go @@ -15,7 +15,7 @@ import ( "strings" "soarca/internal/logger" - "soarca/pkg/models/cacao" + "soarca/pkg/cacao" ) var ( diff --git a/pkg/utils/stix/expression/comparison/comparison_test.go b/pkg/utils/stix/expression/comparison/comparison_test.go index b335fd17..e6eb6d95 100644 --- a/pkg/utils/stix/expression/comparison/comparison_test.go +++ b/pkg/utils/stix/expression/comparison/comparison_test.go @@ -2,7 +2,7 @@ package comparison import ( "errors" - "soarca/pkg/models/cacao" + "soarca/pkg/cacao" "testing" "github.com/go-playground/assert/v2" diff --git a/pkg/utils/stix/expression/comparison/comparison.go b/pkg/utils/stix/expression/comparison/evaluator.go similarity index 99% rename from pkg/utils/stix/expression/comparison/comparison.go rename to pkg/utils/stix/expression/comparison/evaluator.go index b7afdcc8..1723e1ab 100644 --- a/pkg/utils/stix/expression/comparison/comparison.go +++ b/pkg/utils/stix/expression/comparison/evaluator.go @@ -7,7 +7,7 @@ import ( "net/url" "reflect" "soarca/internal/logger" - "soarca/pkg/models/cacao" + "soarca/pkg/cacao" "strconv" "strings" diff --git a/pkg/utils/time/time.go b/pkg/utils/time/clock.go similarity index 100% rename from pkg/utils/time/time.go rename to pkg/utils/time/clock.go diff --git a/pkg/utils/timeout.go b/pkg/utils/timeout.go new file mode 100644 index 00000000..1731e0a8 --- /dev/null +++ b/pkg/utils/timeout.go @@ -0,0 +1,18 @@ +package utils + +import ( + "strconv" + "time" +) + +// defaultStepTimeoutSecondsFallback is used when DEFAULT_STEP_TIMEOUT_SECONDS is unset or invalid. +const defaultStepTimeoutSecondsFallback = 600 + +// DefaultStepTimeout reads DEFAULT_STEP_TIMEOUT_SECONDS with fallback validation. +func DefaultStepTimeout() time.Duration { + seconds, err := strconv.Atoi(GetEnv("DEFAULT_STEP_TIMEOUT_SECONDS", strconv.Itoa(defaultStepTimeoutSecondsFallback))) + if err != nil || seconds <= 0 { + seconds = defaultStepTimeoutSecondsFallback + } + return time.Duration(seconds) * time.Second +} diff --git a/pkg/utils/timeout_test.go b/pkg/utils/timeout_test.go new file mode 100644 index 00000000..98c57bd9 --- /dev/null +++ b/pkg/utils/timeout_test.go @@ -0,0 +1,36 @@ +package utils + +import ( + "os" + "testing" + "time" + + "github.com/go-playground/assert/v2" +) + +func TestDefaultStepTimeoutFallsBackWhenUnset(t *testing.T) { + previous, wasSet := os.LookupEnv("DEFAULT_STEP_TIMEOUT_SECONDS") + os.Unsetenv("DEFAULT_STEP_TIMEOUT_SECONDS") + t.Cleanup(func() { + if wasSet { + os.Setenv("DEFAULT_STEP_TIMEOUT_SECONDS", previous) + } + }) + + assert.Equal(t, DefaultStepTimeout(), 10*time.Minute) +} + +func TestDefaultStepTimeoutReadsEnvVar(t *testing.T) { + t.Setenv("DEFAULT_STEP_TIMEOUT_SECONDS", "120") + assert.Equal(t, DefaultStepTimeout(), 2*time.Minute) +} + +func TestDefaultStepTimeoutFallsBackWhenEnvVarIsInvalid(t *testing.T) { + t.Setenv("DEFAULT_STEP_TIMEOUT_SECONDS", "not-a-number") + assert.Equal(t, DefaultStepTimeout(), 10*time.Minute) +} + +func TestDefaultStepTimeoutFallsBackWhenEnvVarIsNonPositive(t *testing.T) { + t.Setenv("DEFAULT_STEP_TIMEOUT_SECONDS", "0") + assert.Equal(t, DefaultStepTimeout(), 10*time.Minute) +} diff --git a/test/architecture/boundary_test.go b/test/architecture/boundary_test.go new file mode 100644 index 00000000..0dc8eca7 --- /dev/null +++ b/test/architecture/boundary_test.go @@ -0,0 +1,70 @@ +// Package architecture_test enforces the orchestrator/transport boundary. +// +// The orchestrator must stay drivable by any transport (HTTP today, gRPC or a +// CLI later). That only holds if the core never depends on transport concerns, +// so this is asserted mechanically rather than by convention. +package architecture_test + +import ( + "os/exec" + "strings" + "testing" +) + +// corePackages are the orchestrator-side package trees. +var corePackages = []string{ + "soarca/internal/orchestrator/...", + "soarca/internal/runs/...", + "soarca/internal/orchestrator/...", + "soarca/internal/store/...", +} + +// forbiddenInCore are inbound-transport dependencies. Note that net/http is +// deliberately absent: capabilities such as http and openc2 make outbound calls +// and legitimately need it. What the core must never gain is a web framework, +// route registration, or auth middleware. +var forbiddenInCore = []string{ + "github.com/gin-gonic/gin", + "github.com/COSSAS/gauth", + "github.com/swaggo/", + "soarca/internal/transport", + "soarca/internal/transport/http/handlers", +} + +func deps(t *testing.T, packages ...string) []string { + t.Helper() + + args := append([]string{"list", "-deps"}, packages...) + cmd := exec.Command("go", args...) + cmd.Dir = "../.." + + out, err := cmd.Output() + if err != nil { + t.Fatalf("go list -deps failed: %v", err) + } + return strings.Split(strings.TrimSpace(string(out)), "\n") +} + +func TestCoreDoesNotDependOnTransport(t *testing.T) { + for _, dep := range deps(t, corePackages...) { + for _, forbidden := range forbiddenInCore { + if strings.Contains(dep, forbidden) { + t.Errorf("orchestrator core depends on transport concern %q (via %q).\n"+ + "The core must stay drivable by gRPC or a CLI; move this to internal/transport.", + forbidden, dep) + } + } + } +} + +// TestDetectorWorks guards the test above: the transport layer genuinely does +// depend on gin, so if this stops finding it the check has silently stopped +// checking anything. +func TestDetectorWorks(t *testing.T) { + for _, dep := range deps(t, "soarca/internal/transport/...") { + if strings.Contains(dep, "github.com/gin-gonic/gin") { + return + } + } + t.Fatal("expected the transport layer to depend on gin; the boundary detector is not working") +} diff --git a/test/integration/api/api_test.go b/test/integration/api/api_test.go index c557917c..b8c1f415 100644 --- a/test/integration/api/api_test.go +++ b/test/integration/api/api_test.go @@ -5,8 +5,8 @@ import ( "encoding/json" "io" "net/http" - "soarca/internal/controller" - "soarca/pkg/models/api" + "soarca/internal/app" + "soarca/internal/transport/http/schema" "testing" "time" @@ -14,7 +14,7 @@ import ( ) func initializeSoarca(t *testing.T) { - err := controller.Initialize() + err := app.Run() if err != nil { t.Log(err) } diff --git a/test/integration/api/routes/fin_api/fin_api_test.go b/test/integration/api/routes/fin_api/fin_api_test.go new file mode 100644 index 00000000..2ff1e82b --- /dev/null +++ b/test/integration/api/routes/fin_api/fin_api_test.go @@ -0,0 +1,275 @@ +package fin_api_test + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + finservice "soarca/internal/fins" + storagetest "soarca/internal/store/storagetest" + api_routes "soarca/internal/transport/http/handlers" + fin_handler "soarca/internal/transport/http/handlers/fin" + "soarca/internal/workflow/capability/fin/queue" + "soarca/pkg/fins/protocol" + "soarca/pkg/utils/guid" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" +) + +const registrationToken = "test-registration-token" + +// newFinTestApp wires the real registry and work service over in-memory storage, +// so these tests cover the actual route/middleware/service composition. +func newFinTestApp(t *testing.T, regToken string) *gin.Engine { + t.Helper() + + store := storagetest.New(t) + q := queue.New() + t.Cleanup(q.Close) + + registry := finservice.NewRegistry( + store.Fins(), + finservice.RegistryConfig{RegistrationToken: regToken, StaleAfter: 10 * time.Second}, + new(guid.Guid), + ) + workService := finservice.NewWorkService( + store.Fins(), + q, + // Long poll timeout kept at 1s so "no work available" returns quickly. + finservice.WorkServiceConfig{LongPollTimeoutSeconds: 1, JobLeaseSeconds: 60}, + ) + + handler := fin_handler.NewFinHandler(registry, workService, fin_handler.Config{ + RegistrationToken: regToken, + PollIntervalSeconds: 5, + LongPollTimeoutSeconds: 1, + JobLeaseSeconds: 60, + StaleAfter: 10 * time.Second, + }) + + gin.SetMode(gin.TestMode) + app := gin.New() + api_routes.FinPublic(app, handler) + api_routes.FinAdmin(app, handler) + return app +} + +func doRequest(app *gin.Engine, method, path, token string, body any) *httptest.ResponseRecorder { + var reader *bytes.Reader + if body != nil { + encoded, _ := json.Marshal(body) + reader = bytes.NewReader(encoded) + } else { + reader = bytes.NewReader(nil) + } + + request, _ := http.NewRequest(method, path, reader) + if body != nil { + request.Header.Set("Content-Type", "application/json") + } + if token != "" { + request.Header.Set("Authorization", "Bearer "+token) + } + + recorder := httptest.NewRecorder() + app.ServeHTTP(recorder, request) + return recorder +} + +func registerFin(t *testing.T, app *gin.Engine) fin.RegisterResponse { + t.Helper() + + recorder := doRequest(app, "POST", "/fin/register", "", fin.RegisterRequest{ + RegistrationToken: registrationToken, + DisplayName: "test-fin", + Capabilities: []fin.Capability{{Type: "soarca-fin-test"}}, + }) + assert.Equal(t, http.StatusCreated, recorder.Code) + + var response fin.RegisterResponse + if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + t.Fatalf("could not unmarshal register response: %v", err) + } + return response +} + +func TestRegisterFin(t *testing.T) { + app := newFinTestApp(t, registrationToken) + + response := registerFin(t, app) + + assert.NotEmpty(t, response.FinId) + assert.NotEmpty(t, response.FinToken) + assert.Equal(t, 5, response.PollIntervalSeconds) + assert.Equal(t, 1, response.LongPollTimeoutSeconds) + assert.Equal(t, 60, response.JobLeaseSeconds) +} + +func TestRegisterFinRejectsWrongRegistrationToken(t *testing.T) { + app := newFinTestApp(t, registrationToken) + + recorder := doRequest(app, "POST", "/fin/register", "", fin.RegisterRequest{ + RegistrationToken: "wrong-token", + Capabilities: []fin.Capability{{Type: "soarca-fin-test"}}, + }) + + assert.Equal(t, http.StatusForbidden, recorder.Code) +} + +func TestRegisterFinRejectsMissingCapabilities(t *testing.T) { + app := newFinTestApp(t, registrationToken) + + recorder := doRequest(app, "POST", "/fin/register", "", fin.RegisterRequest{ + RegistrationToken: registrationToken, + }) + + assert.Equal(t, http.StatusBadRequest, recorder.Code) +} + +func TestRegisterFinUnavailableWhenRegistrationTokenNotConfigured(t *testing.T) { + app := newFinTestApp(t, "") + + recorder := doRequest(app, "POST", "/fin/register", "", fin.RegisterRequest{ + RegistrationToken: registrationToken, + Capabilities: []fin.Capability{{Type: "soarca-fin-test"}}, + }) + + assert.Equal(t, http.StatusServiceUnavailable, recorder.Code) +} + +func TestPollRequiresAuthorizationHeader(t *testing.T) { + app := newFinTestApp(t, registrationToken) + + recorder := doRequest(app, "POST", "/fin/poll", "", nil) + + assert.Equal(t, http.StatusUnauthorized, recorder.Code) +} + +func TestPollRejectsUnknownToken(t *testing.T) { + app := newFinTestApp(t, registrationToken) + + recorder := doRequest(app, "POST", "/fin/poll", "not-a-real-token", nil) + + assert.Equal(t, http.StatusUnauthorized, recorder.Code) +} + +func TestPollReturnsNoContentWhenNoWorkAvailable(t *testing.T) { + app := newFinTestApp(t, registrationToken) + registered := registerFin(t, app) + + recorder := doRequest(app, "POST", "/fin/poll", registered.FinToken, fin.PollRequest{ConcurrencyAvailable: 1}) + + assert.Equal(t, http.StatusNoContent, recorder.Code) +} + +func TestSubmitResultRejectsMalformedJobId(t *testing.T) { + app := newFinTestApp(t, registrationToken) + registered := registerFin(t, app) + + recorder := doRequest(app, "PUT", "/fin/jobs/not-a-uuid", registered.FinToken, fin.ResultRequest{ + JobResult: fin.JobResult{State: fin.JobStateSuccess}, + }) + + assert.Equal(t, http.StatusBadRequest, recorder.Code) +} + +func TestSubmitResultRejectsInvalidState(t *testing.T) { + app := newFinTestApp(t, registrationToken) + registered := registerFin(t, app) + + recorder := doRequest(app, "PUT", "/fin/jobs/6ba7b810-9dad-11d1-80b4-00c04fd430c0", registered.FinToken, + fin.ResultRequest{JobResult: fin.JobResult{State: "not-a-state"}}) + + assert.Equal(t, http.StatusBadRequest, recorder.Code) +} + +func TestSubmitResultForUnknownJobIsNotFound(t *testing.T) { + app := newFinTestApp(t, registrationToken) + registered := registerFin(t, app) + + recorder := doRequest(app, "PUT", "/fin/jobs/6ba7b810-9dad-11d1-80b4-00c04fd430c0", registered.FinToken, + fin.ResultRequest{JobResult: fin.JobResult{State: fin.JobStateSuccess}}) + + assert.Equal(t, http.StatusNotFound, recorder.Code) +} + +func TestListFins(t *testing.T) { + app := newFinTestApp(t, registrationToken) + registered := registerFin(t, app) + + recorder := doRequest(app, "GET", "/fin/", "", nil) + assert.Equal(t, http.StatusOK, recorder.Code) + + var response fin.ListResponse + if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + t.Fatalf("could not unmarshal list response: %v", err) + } + + assert.Equal(t, 1, len(response.Fins)) + assert.Equal(t, registered.FinId, response.Fins[0].FinId) + assert.Equal(t, "test-fin", response.Fins[0].DisplayName) + assert.Equal(t, false, response.Fins[0].Stale) +} + +// The fin token hash is a credential; Record doubles as the admin wire type and +// only a json:"-" tag keeps the hash off the wire. +func TestListFinsDoesNotLeakTokenHash(t *testing.T) { + app := newFinTestApp(t, registrationToken) + registered := registerFin(t, app) + + recorder := doRequest(app, "GET", "/fin/", "", nil) + body := recorder.Body.String() + + assert.Equal(t, false, strings.Contains(body, "fin_token_hash")) + assert.Equal(t, false, strings.Contains(body, "FinTokenHash")) + assert.Equal(t, false, strings.Contains(body, registered.FinToken)) +} + +func TestGetFinById(t *testing.T) { + app := newFinTestApp(t, registrationToken) + registered := registerFin(t, app) + + recorder := doRequest(app, "GET", "/fin/"+registered.FinId, "", nil) + assert.Equal(t, http.StatusOK, recorder.Code) + + var record fin.Record + if err := json.Unmarshal(recorder.Body.Bytes(), &record); err != nil { + t.Fatalf("could not unmarshal fin record: %v", err) + } + assert.Equal(t, registered.FinId, record.FinId) +} + +func TestGetUnknownFinIsNotFound(t *testing.T) { + app := newFinTestApp(t, registrationToken) + + recorder := doRequest(app, "GET", "/fin/does-not-exist", "", nil) + + assert.Equal(t, http.StatusNotFound, recorder.Code) +} + +func TestDeleteFin(t *testing.T) { + app := newFinTestApp(t, registrationToken) + registered := registerFin(t, app) + + recorder := doRequest(app, "DELETE", "/fin/"+registered.FinId, "", nil) + assert.Equal(t, http.StatusNoContent, recorder.Code) + + recorder = doRequest(app, "GET", "/fin/"+registered.FinId, "", nil) + assert.Equal(t, http.StatusNotFound, recorder.Code) +} + +func TestUnregisterFin(t *testing.T) { + app := newFinTestApp(t, registrationToken) + registered := registerFin(t, app) + + recorder := doRequest(app, "DELETE", "/fin/", registered.FinToken, nil) + assert.Equal(t, http.StatusNoContent, recorder.Code) + + recorder = doRequest(app, "POST", "/fin/poll", registered.FinToken, nil) + assert.Equal(t, http.StatusUnauthorized, recorder.Code) +} diff --git a/test/integration/api/routes/manual_api/manual_api_test.go b/test/integration/api/routes/manual_api/manual_api_test.go index b94fbe2e..25daeae4 100644 --- a/test/integration/api/routes/manual_api/manual_api_test.go +++ b/test/integration/api/routes/manual_api/manual_api_test.go @@ -6,13 +6,14 @@ import ( "errors" "net/http" "net/http/httptest" - api_routes "soarca/pkg/api" - manual_api "soarca/pkg/api/manual" - apiModel "soarca/pkg/models/api" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" - "soarca/pkg/models/manual" - "soarca/test/unittest/mocks/mock_interaction_storage" + api_routes "soarca/internal/transport/http/handlers" + manual_api "soarca/internal/transport/http/handlers/manual" + "soarca/internal/workflow/capability" + apiModel "soarca/internal/transport/http/schema" + "soarca/pkg/cacao" + "soarca/internal/manual/model" + "soarca/internal/runs/model" + "soarca/test/unittest/mocks/mock_manual_inbox_storage" "strings" "testing" @@ -22,8 +23,8 @@ import ( ) func TestGetPendingCommandsCalled(t *testing.T) { - mock_interaction_storage := mock_interaction_storage.MockInteractionStorage{} - manualApiHandler := manual_api.NewManualHandler(&mock_interaction_storage) + mock_manual_inbox_storage := mock_manual_inbox_storage.MockInboxStorage{} + manualApiHandler := manual_api.NewManualHandler(&mock_manual_inbox_storage) app := gin.New() gin.SetMode(gin.DebugMode) @@ -31,7 +32,7 @@ func TestGetPendingCommandsCalled(t *testing.T) { recorder := httptest.NewRecorder() api_routes.ManualRoutes(app, manualApiHandler) - mock_interaction_storage.On("GetPendingCommands").Return([]manual.CommandInfo{}, nil) + mock_manual_inbox_storage.On("GetPendingCommands").Return([]manual.CommandInfo{}, nil) request, err := http.NewRequest("GET", "/manual/", nil) if err != nil { @@ -44,12 +45,12 @@ func TestGetPendingCommandsCalled(t *testing.T) { assert.Equal(t, expectedString, recorder.Body.String()) assert.Equal(t, 200, recorder.Code) - mock_interaction_storage.AssertExpectations(t) + mock_manual_inbox_storage.AssertExpectations(t) } func TestGetPendingCommandCalled(t *testing.T) { - mock_interaction_storage := mock_interaction_storage.MockInteractionStorage{} - manualApiHandler := manual_api.NewManualHandler(&mock_interaction_storage) + mock_manual_inbox_storage := mock_manual_inbox_storage.MockInboxStorage{} + manualApiHandler := manual_api.NewManualHandler(&mock_manual_inbox_storage) app := gin.New() gin.SetMode(gin.DebugMode) @@ -57,19 +58,22 @@ func TestGetPendingCommandCalled(t *testing.T) { recorder := httptest.NewRecorder() api_routes.ManualRoutes(app, manualApiHandler) testExecId := "50b6d52c-6efc-4516-a242-dfbc5c89d421" - testStepId := "61a4d52c-6efc-4516-a242-dfbc5c89d312" - path := "/manual/" + testExecId + "/" + testStepId - executionMetadata := execution.Metadata{ - ExecutionId: uuid.MustParse(testExecId), StepId: testStepId, + testStepRunId := "71a4d52c-6efc-4516-a242-dfbc5c89d999" + path := "/manual/" + testExecId + "/" + testStepRunId + runMetadata := run.Metadata{ + RunId: uuid.MustParse(testExecId), StepRunId: uuid.MustParse(testStepRunId), } - testEmptyResponsePendingCommand := apiModel.InteractionCommandData{ - Type: "manual-command-info", - ExecutionId: "00000000-0000-0000-0000-000000000000", + testEmptyResponsePendingCommand := apiModel.PendingCommandData{ + Type: "manual-command-info", + RunId: "00000000-0000-0000-0000-000000000000", + StepRunId: "00000000-0000-0000-0000-000000000000", + Commands: []apiModel.ManualCommand{}, + Targets: []capability.ResolvedTarget{}, } emptyCommandInfoList := manual.CommandInfo{} - mock_interaction_storage.On("GetPendingCommand", executionMetadata).Return(emptyCommandInfoList, nil) + mock_manual_inbox_storage.On("GetPendingCommand", runMetadata).Return(emptyCommandInfoList, nil) request, err := http.NewRequest("GET", path, nil) if err != nil { @@ -89,12 +93,15 @@ func TestGetPendingCommandCalled(t *testing.T) { assert.Equal(t, expectedString, recorder.Body.String()) assert.Equal(t, 200, recorder.Code) - mock_interaction_storage.AssertExpectations(t) + mock_manual_inbox_storage.AssertExpectations(t) } -func TestPostContinueCalled(t *testing.T) { - mock_interaction_storage := mock_interaction_storage.MockInteractionStorage{} - manualApiHandler := manual_api.NewManualHandler(&mock_interaction_storage) +// PUT /manual/{exec_id}/{step_run_id} resolves the pending command +// identified by the path - the same resource GET identifies - rather than a +// generic POST /manual/continue carrying its own ids in the body. +func TestPutContinueCalled(t *testing.T) { + mock_manual_inbox_storage := mock_manual_inbox_storage.MockInboxStorage{} + manualApiHandler := manual_api.NewManualHandler(&mock_manual_inbox_storage) app := gin.New() gin.SetMode(gin.DebugMode) @@ -103,14 +110,28 @@ func TestPostContinueCalled(t *testing.T) { api_routes.ManualRoutes(app, manualApiHandler) testExecId := "50b6d52c-6efc-4516-a242-dfbc5c89d421" testStepId := "61a4d52c-6efc-4516-a242-dfbc5c89d312" + testStepRunId := "71a4d52c-6efc-4516-a242-dfbc5c89d999" testPlaybookId := "21a4d52c-6efc-4516-a242-dfbc5c89d312" - path := "/manual/continue" + path := "/manual/" + testExecId + "/" + testStepRunId + + // The metadata built from the path alone, used to look up the pending + // command. + lookupMetadata := run.Metadata{ + RunId: uuid.MustParse(testExecId), + StepRunId: uuid.MustParse(testStepRunId), + } + // The full metadata of the pending command as returned by the lookup; + // this - not the (ids-free) request body - is what flows into + // PostContinue. + fullMetadata := run.Metadata{ + RunId: uuid.MustParse(testExecId), + StepId: testStepId, + StepRunId: uuid.MustParse(testStepRunId), + PlaybookId: testPlaybookId, + } testManualUpdatePayload := apiModel.ManualOutArgsUpdatePayload{ Type: "manual-step-response", - ExecutionId: testExecId, - StepId: testStepId, - PlaybookId: testPlaybookId, ResponseStatus: "success", ResponseOutArgs: cacao.Variables{ "testvar": { @@ -121,12 +142,10 @@ func TestPostContinueCalled(t *testing.T) { }, } - testManualResponse := manual.InteractionResponse{ - Metadata: execution.Metadata{ - ExecutionId: uuid.MustParse(testExecId), - StepId: testStepId, - PlaybookId: testPlaybookId, - }, + pendingCommand := manual.CommandInfo{Metadata: fullMetadata} + + testManualResponse := manual.Response{ + Metadata: fullMetadata, ResponseStatus: "success", OutArgsVariables: cacao.Variables{ "testvar": { @@ -141,9 +160,10 @@ func TestPostContinueCalled(t *testing.T) { t.Fatalf("Error marshalling JSON: %v", err) } - mock_interaction_storage.On("PostContinue", testManualResponse).Return(nil) + mock_manual_inbox_storage.On("GetPendingCommand", lookupMetadata).Return(pendingCommand, nil) + mock_manual_inbox_storage.On("PostContinue", testManualResponse).Return(nil) - request, err := http.NewRequest("POST", path, bytes.NewBuffer(jsonData)) + request, err := http.NewRequest("PUT", path, bytes.NewBuffer(jsonData)) if err != nil { t.Fail() } @@ -152,12 +172,12 @@ func TestPostContinueCalled(t *testing.T) { t.Log(recorder.Body.String()) assert.Equal(t, 200, recorder.Code) - mock_interaction_storage.AssertExpectations(t) + mock_manual_inbox_storage.AssertExpectations(t) } -func TestPostContinueFailsOnNonMatchingOutArgNames(t *testing.T) { - mock_interaction_storage := mock_interaction_storage.MockInteractionStorage{} - manualApiHandler := manual_api.NewManualHandler(&mock_interaction_storage) +func TestPutContinueFailsOnNonMatchingOutArgNames(t *testing.T) { + mock_manual_inbox_storage := mock_manual_inbox_storage.MockInboxStorage{} + manualApiHandler := manual_api.NewManualHandler(&mock_manual_inbox_storage) app := gin.New() gin.SetMode(gin.DebugMode) @@ -165,15 +185,11 @@ func TestPostContinueFailsOnNonMatchingOutArgNames(t *testing.T) { recorder := httptest.NewRecorder() api_routes.ManualRoutes(app, manualApiHandler) testExecId := "50b6d52c-6efc-4516-a242-dfbc5c89d421" - testStepId := "61a4d52c-6efc-4516-a242-dfbc5c89d312" - testPlaybookId := "21a4d52c-6efc-4516-a242-dfbc5c89d312" - path := "/manual/continue" + testStepRunId := "71a4d52c-6efc-4516-a242-dfbc5c89d999" + path := "/manual/" + testExecId + "/" + testStepRunId testManualUpdatePayload := apiModel.ManualOutArgsUpdatePayload{ Type: "manual-step-response", - ExecutionId: testExecId, - StepId: testStepId, - PlaybookId: testPlaybookId, ResponseStatus: "success", ResponseOutArgs: cacao.Variables{ "__this_var__": { @@ -189,7 +205,7 @@ func TestPostContinueFailsOnNonMatchingOutArgNames(t *testing.T) { t.Fatalf("Error marshalling JSON: %v", err) } - request, err := http.NewRequest("POST", path, bytes.NewBuffer(manualUpdatePayloadJson)) + request, err := http.NewRequest("PUT", path, bytes.NewBuffer(manualUpdatePayloadJson)) if err != nil { t.Fail() } @@ -199,6 +215,6 @@ func TestPostContinueFailsOnNonMatchingOutArgNames(t *testing.T) { app.ServeHTTP(recorder, request) t.Log(recorder.Body.String()) assert.Equal(t, 400, recorder.Code) - mock_interaction_storage.AssertExpectations(t) + mock_manual_inbox_storage.AssertExpectations(t) assert.Equal(t, true, strings.Contains(recorder.Body.String(), expectedErr.Error())) } diff --git a/test/integration/api/routes/playbook_api/playbook_api_test.go b/test/integration/api/routes/playbook_api/playbook_api_test.go index 89cd9b50..ddc0eab3 100644 --- a/test/integration/api/routes/playbook_api/playbook_api_test.go +++ b/test/integration/api/routes/playbook_api/playbook_api_test.go @@ -8,17 +8,18 @@ import ( "net/http" "net/http/httptest" "os" - api_routes "soarca/pkg/api" - "soarca/pkg/models/api" - "soarca/pkg/models/cacao" - "soarca/pkg/models/decoder" + api_routes "soarca/internal/transport/http/handlers" + "soarca/internal/transport/http/schema" + "soarca/pkg/cacao" + "soarca/internal/playbooks/decoder" "testing" - mock_database_controller "soarca/test/unittest/mocks/mock_controller/database" + playbookservice "soarca/internal/playbooks/library" mock_playbook "soarca/test/unittest/mocks/mock_playbook_database" "github.com/gin-gonic/gin" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" ) const jsonTestPlayBookMeta = `{ @@ -41,10 +42,9 @@ func close(file *os.File) { } } -func TestGetPlaybookMetas(t *testing.T) { +func TestListMeta(t *testing.T) { app := gin.New() - mockController := new(mock_database_controller.Mock_Controller) mockPlaybook := new(mock_playbook.MockPlaybook) var dummyPlaybookMeta api.PlaybookMeta @@ -66,12 +66,10 @@ func TestGetPlaybookMetas(t *testing.T) { t.Fail() return } - mockController.On("GetDatabaseInstance").Return(mockPlaybook) - - mockPlaybook.On("GetPlaybookMetas").Return(dummyPlaybookMetas, nil) + mockPlaybook.On("ListMeta", mock.Anything).Return(dummyPlaybookMetas, nil) w := httptest.NewRecorder() - api_routes.PlaybookRoutes(app, mockController) + api_routes.PlaybookRoutesWithService(app, playbookservice.New(mockPlaybook)) req, _ := http.NewRequest("GET", "/playbook/meta/", nil) app.ServeHTTP(w, req) @@ -80,7 +78,7 @@ func TestGetPlaybookMetas(t *testing.T) { assert.JSONEq(t, string(marshalledDummyPlayBookMetas), w.Body.String()) } -func TestGetPlaybooks(t *testing.T) { +func TestList(t *testing.T) { jsonFile, err := os.Open("../playbook.json") if err != nil { fmt.Println(err) @@ -92,9 +90,7 @@ func TestGetPlaybooks(t *testing.T) { app := gin.New() gin.SetMode(gin.DebugMode) - mockController := new(mock_database_controller.Mock_Controller) mockPlaybook := new(mock_playbook.MockPlaybook) - mockController.On("GetDatabaseInstance").Return(mockPlaybook) dummyPlaybook := decoder.DecodeValidate(byteValue) if dummyPlaybook == nil { fmt.Println("got an nil playbook pointer") @@ -113,9 +109,9 @@ func TestGetPlaybooks(t *testing.T) { t.Fail() return } - mockPlaybook.On("GetPlaybooks").Return(playbooks, nil) + mockPlaybook.On("List", mock.Anything).Return(playbooks, nil) w := httptest.NewRecorder() - api_routes.PlaybookRoutes(app, mockController) + api_routes.PlaybookRoutesWithService(app, playbookservice.New(mockPlaybook)) req, _ := http.NewRequest("GET", "/playbook/", nil) app.ServeHTTP(w, req) @@ -134,11 +130,9 @@ func TestGetPlaybookByID(t *testing.T) { byteValue, _ := io.ReadAll(jsonFile) app := gin.New() - mockController := new(mock_database_controller.Mock_Controller) mockPlaybook := new(mock_playbook.MockPlaybook) - mockController.On("GetDatabaseInstance").Return(mockPlaybook) dummyPlaybook := decoder.DecodeValidate(byteValue) - mockPlaybook.On("Read", dummyPlaybook.ID).Return(*dummyPlaybook, nil) + mockPlaybook.On("Get", mock.Anything, dummyPlaybook.ID).Return(*dummyPlaybook, nil) marshalledDummyPlaybook, err := json.Marshal(dummyPlaybook) if err != nil { fmt.Println("Failed to marshall dummy JSON:", err) @@ -147,7 +141,7 @@ func TestGetPlaybookByID(t *testing.T) { } w := httptest.NewRecorder() - api_routes.PlaybookRoutes(app, mockController) + api_routes.PlaybookRoutesWithService(app, playbookservice.New(mockPlaybook)) req, _ := http.NewRequest("GET", fmt.Sprintf("/playbook/%s", dummyPlaybook.ID), nil) app.ServeHTTP(w, req) @@ -166,9 +160,7 @@ func TestPostPlaybook(t *testing.T) { byteValue, _ := io.ReadAll(jsonFile) app := gin.New() - mockController := new(mock_database_controller.Mock_Controller) mockPlaybook := new(mock_playbook.MockPlaybook) - mockController.On("GetDatabaseInstance").Return(mockPlaybook) dummyPlaybook := decoder.DecodeValidate(byteValue) if dummyPlaybook == nil { @@ -182,11 +174,10 @@ func TestPostPlaybook(t *testing.T) { t.Fail() return } - pointerDummyObject := []byte(marshalledDummyPlaybook) - mockPlaybook.On("Create", &pointerDummyObject).Return(*dummyPlaybook, nil) + mockPlaybook.On("Create", mock.Anything, mock.Anything).Return(nil) w := httptest.NewRecorder() - api_routes.PlaybookRoutes(app, mockController) + api_routes.PlaybookRoutesWithService(app, playbookservice.New(mockPlaybook)) req, _ := http.NewRequest("POST", "/playbook/", bytes.NewBuffer(marshalledDummyPlaybook)) app.ServeHTTP(w, req) @@ -205,9 +196,7 @@ func TestDeletePlaybook(t *testing.T) { byteValue, _ := io.ReadAll(jsonFile) app := gin.New() - mockController := new(mock_database_controller.Mock_Controller) mockPlaybook := new(mock_playbook.MockPlaybook) - mockController.On("GetDatabaseInstance").Return(mockPlaybook) dummyPlaybook := decoder.DecodeValidate(byteValue) if dummyPlaybook == nil { @@ -215,9 +204,9 @@ func TestDeletePlaybook(t *testing.T) { t.Fail() return } - mockPlaybook.On("Delete", dummyPlaybook.ID).Return(nil) + mockPlaybook.On("Delete", mock.Anything, dummyPlaybook.ID).Return(nil) w := httptest.NewRecorder() - api_routes.PlaybookRoutes(app, mockController) + api_routes.PlaybookRoutesWithService(app, playbookservice.New(mockPlaybook)) req, _ := http.NewRequest("DELETE", fmt.Sprintf("/playbook/%s", dummyPlaybook.ID), nil) app.ServeHTTP(w, req) assert.Equal(t, 200, w.Code) @@ -233,9 +222,7 @@ func TestUpdatePlaybook(t *testing.T) { byteValue, _ := io.ReadAll(jsonFile) app := gin.New() - mockController := new(mock_database_controller.Mock_Controller) mockPlaybook := new(mock_playbook.MockPlaybook) - mockController.On("GetDatabaseInstance").Return(mockPlaybook) dummyPlaybook := decoder.DecodeValidate(byteValue) if dummyPlaybook == nil { @@ -249,11 +236,10 @@ func TestUpdatePlaybook(t *testing.T) { t.Fail() return } - pointerDummyObject := []byte(marshalledDummyPlaybook) - mockPlaybook.On("Update", dummyPlaybook.ID, &pointerDummyObject).Return(*dummyPlaybook, nil) + mockPlaybook.On("Update", mock.Anything, mock.Anything).Return(nil) w := httptest.NewRecorder() - api_routes.PlaybookRoutes(app, mockController) + api_routes.PlaybookRoutesWithService(app, playbookservice.New(mockPlaybook)) req, _ := http.NewRequest("PUT", fmt.Sprintf("/playbook/%s", dummyPlaybook.ID), bytes.NewBuffer(marshalledDummyPlaybook)) app.ServeHTTP(w, req) diff --git a/test/integration/api/routes/reporter_api/reporter_api_invocation_test.go b/test/integration/api/routes/reporter_api/reporter_api_invocation_test.go index 40e79c6c..e738f5bc 100644 --- a/test/integration/api/routes/reporter_api/reporter_api_invocation_test.go +++ b/test/integration/api/routes/reporter_api/reporter_api_invocation_test.go @@ -5,27 +5,29 @@ import ( "fmt" "net/http" "net/http/httptest" - api_routes "soarca/pkg/api" - api_model "soarca/pkg/models/api" - cache_model "soarca/pkg/models/cache" - mock_cache "soarca/test/unittest/mocks/mock_cache" + api_routes "soarca/internal/transport/http/handlers" + api_model "soarca/internal/transport/http/schema" + runstate_model "soarca/internal/runs/state" + mock_runstate "soarca/test/unittest/mocks/mock_runstate" "testing" + runsservice "soarca/internal/runs" + "github.com/google/uuid" "github.com/gin-gonic/gin" "github.com/go-playground/assert/v2" ) -func TestGetExecutionsInvocation(t *testing.T) { - mock_cache_reporter := &mock_cache.Mock_Cache{} - mock_cache_reporter.On("GetExecutions").Return([]cache_model.ExecutionEntry{}, nil) +func TestGetRunsInvocation(t *testing.T) { + mock_runstate_reporter := &mock_runstate.MockRunState{} + mock_runstate_reporter.On("GetRuns").Return([]runstate_model.RunEntry{}, nil) app := gin.New() gin.SetMode(gin.DebugMode) recorder := httptest.NewRecorder() - api_routes.ReporterRoutes(app, mock_cache_reporter) + api_routes.ReporterRoutesWithService(app, runsservice.New(nil, nil, mock_runstate_reporter)) request, err := http.NewRequest("GET", "/reporter/", nil) if err != nil { @@ -37,28 +39,29 @@ func TestGetExecutionsInvocation(t *testing.T) { assert.Equal(t, expectedString, recorder.Body.String()) assert.Equal(t, 200, recorder.Code) - mock_cache_reporter.AssertExpectations(t) + mock_runstate_reporter.AssertExpectations(t) } -func TestGetExecutionReportInvocation(t *testing.T) { - mock_cache_reporter := &mock_cache.Mock_Cache{} +func TestGetRunReportInvocation(t *testing.T) { + mock_runstate_reporter := &mock_runstate.MockRunState{} app := gin.New() gin.SetMode(gin.DebugMode) recorder := httptest.NewRecorder() - api_routes.ReporterRoutes(app, mock_cache_reporter) + api_routes.ReporterRoutesWithService(app, runsservice.New(nil, nil, mock_runstate_reporter)) - executionId0, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c0") + runId0, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c0") expectedCache := `{ - "ExecutionId":"6ba7b810-9dad-11d1-80b4-00c04fd430c0", + "RunId":"6ba7b810-9dad-11d1-80b4-00c04fd430c0", "PlaybookId":"test", "Started":"2014-11-12T11:45:26.371Z", "Ended":"0001-01-01T00:00:00Z", "StepResults":{ - "action--test":{ - "ExecutionId":"6ba7b810-9dad-11d1-80b4-00c04fd430c0", + "6ba7b810-9dad-11d1-80b4-00c04fd430c9":{ + "RunId":"6ba7b810-9dad-11d1-80b4-00c04fd430c0", "StepId":"action--test", + "StepRunId":"6ba7b810-9dad-11d1-80b4-00c04fd430c9", "Started":"2014-11-12T11:45:26.371Z", "Ended":"2014-11-12T11:45:26.371Z", "Variables":{ @@ -77,7 +80,7 @@ func TestGetExecutionReportInvocation(t *testing.T) { "PlaybookResult":null, "Status":2 }` - expectedCacheData := cache_model.ExecutionEntry{} + expectedCacheData := runstate_model.RunEntry{} err := json.Unmarshal([]byte(expectedCache), &expectedCacheData) if err != nil { t.Log(err) @@ -85,9 +88,9 @@ func TestGetExecutionReportInvocation(t *testing.T) { t.Fail() } - mock_cache_reporter.On("GetExecutionReport", executionId0).Return(expectedCacheData, nil) + mock_runstate_reporter.On("GetRunReport", runId0).Return(expectedCacheData, nil) - request, err := http.NewRequest("GET", fmt.Sprintf("/reporter/%s", executionId0), nil) + request, err := http.NewRequest("GET", fmt.Sprintf("/reporter/%s", runId0), nil) if err != nil { t.Log(err) t.Fail() @@ -95,21 +98,22 @@ func TestGetExecutionReportInvocation(t *testing.T) { app.ServeHTTP(recorder, request) expectedResponse := `{ - "type":"execution_status", - "execution_id":"6ba7b810-9dad-11d1-80b4-00c04fd430c0", + "type":"run_status", + "run_id":"6ba7b810-9dad-11d1-80b4-00c04fd430c0", "playbook_id":"test", "started":"2014-11-12T11:45:26.371Z", "ended":"0001-01-01T00:00:00Z", "status":"ongoing", "status_text":"this playbook is currently being executed", "step_results":{ - "action--test":{ - "execution_id":"6ba7b810-9dad-11d1-80b4-00c04fd430c0", + "6ba7b810-9dad-11d1-80b4-00c04fd430c9":{ + "run_id":"6ba7b810-9dad-11d1-80b4-00c04fd430c0", "step_id": "action--test", + "step_run_id": "6ba7b810-9dad-11d1-80b4-00c04fd430c9", "started": "2014-11-12T11:45:26.371Z", "ended": "2014-11-12T11:45:26.371Z", "status": "successfully_executed", - "status_text": "step execution completed successfully", + "status_text": "step run completed successfully", "Variables":{ "var1":{ "type":"string", @@ -118,13 +122,13 @@ func TestGetExecutionReportInvocation(t *testing.T) { } }, "commands_b64" : [], - "automated_execution" : true, + "automated_run" : true, "executed_by" : "soarca" } }, "request_interval":5 }` - expectedResponseData := api_model.PlaybookExecutionReport{} + expectedResponseData := api_model.PlaybookRunReport{} err = json.Unmarshal([]byte(expectedResponse), &expectedResponseData) if err != nil { t.Log(err) @@ -132,7 +136,7 @@ func TestGetExecutionReportInvocation(t *testing.T) { t.Fail() } - receivedData := api_model.PlaybookExecutionReport{} + receivedData := api_model.PlaybookRunReport{} err = json.Unmarshal(recorder.Body.Bytes(), &receivedData) if err != nil { t.Log(err) @@ -145,5 +149,5 @@ func TestGetExecutionReportInvocation(t *testing.T) { t.Log("received response") t.Log(receivedData) assert.Equal(t, expectedResponseData, receivedData) - mock_cache_reporter.AssertExpectations(t) + mock_runstate_reporter.AssertExpectations(t) } diff --git a/test/integration/api/routes/reporter_api/reporter_api_test.go b/test/integration/api/routes/reporter_api/reporter_api_test.go index e3444c6e..939c6dea 100644 --- a/test/integration/api/routes/reporter_api/reporter_api_test.go +++ b/test/integration/api/routes/reporter_api/reporter_api_test.go @@ -5,15 +5,18 @@ import ( "fmt" "net/http" "net/http/httptest" - api_model "soarca/pkg/models/api" - "soarca/pkg/models/cacao" - cache_model "soarca/pkg/models/cache" - "soarca/pkg/reporting/reporter/downstream_reporter/cache" + api_model "soarca/internal/transport/http/schema" + "soarca/pkg/cacao" + runstate_model "soarca/internal/runs/state" + "soarca/internal/runs/model" + "soarca/internal/reporting/reporter/downstream_reporter/runstate" mock_time "soarca/test/unittest/mocks/mock_utils/time" "testing" "time" - api_routes "soarca/pkg/api" + api_routes "soarca/internal/transport/http/handlers" + + runsservice "soarca/internal/runs" "github.com/google/uuid" @@ -21,9 +24,9 @@ import ( "github.com/go-playground/assert/v2" ) -func TestGetExecutions(t *testing.T) { +func TestGetRuns(t *testing.T) { mock_time := new(mock_time.MockTime) - cacheReporter := cache.New(mock_time, 10) + cacheReporter := runstate.New(mock_time, 10) expectedCommand := cacao.Command{ Type: "ssh", @@ -81,14 +84,14 @@ func TestGetExecutions(t *testing.T) { Workflow: map[string]cacao.Step{step1.ID: step1, end.ID: end}, } - executionId0 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c0") - executionId1 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c1") - executionId2 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c2") + runId0 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c0") + runId1 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c1") + runId2 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c2") - executionIds := []uuid.UUID{ - executionId0, - executionId1, - executionId2, + runIds := []uuid.UUID{ + runId0, + runId1, + runId2, } layout := "2006-01-02T15:04:05.000Z" @@ -99,37 +102,37 @@ func TestGetExecutions(t *testing.T) { expectedStarted, _ := time.Parse(layout, str) expectedEnded, _ := time.Parse(layout, "0001-01-01T00:00:00Z") - expectedStatus := cache_model.Ongoing.String() - expectedStatusText, _ := api_model.GetCacheStatusText(expectedStatus, "playbook") + expectedStatus := runstate_model.Ongoing.String() + expectedStatusText, _ := api_model.GetRunStatusText(expectedStatus, "playbook") - expectedExecutionsReport := []api_model.PlaybookExecutionReport{} - for _, executionId := range executionIds { - t.Log(executionId) - entry := api_model.PlaybookExecutionReport{ - Type: "execution_status", - ExecutionId: executionId.String(), + expectedRunsReport := []api_model.PlaybookRunReport{} + for _, runId := range runIds { + t.Log(runId) + entry := api_model.PlaybookRunReport{ + Type: "run_status", + RunId: runId.String(), PlaybookId: "test", Name: "ssh-test", Started: expectedStarted, Ended: expectedEnded, Status: expectedStatus, StatusText: expectedStatusText, - StepResults: map[string]api_model.StepExecutionReport{}, + StepResults: map[string]api_model.StepRunReport{}, RequestInterval: 5, } - expectedExecutionsReport = append(expectedExecutionsReport, entry) + expectedRunsReport = append(expectedRunsReport, entry) } - err := cacheReporter.ReportWorkflowStart(executionId0, playbook, mock_time.Now()) + err := cacheReporter.ReportWorkflowStart(runId0, playbook, mock_time.Now()) if err != nil { t.Fail() } - err = cacheReporter.ReportWorkflowStart(executionId1, playbook, mock_time.Now()) + err = cacheReporter.ReportWorkflowStart(runId1, playbook, mock_time.Now()) if err != nil { t.Fail() } - err = cacheReporter.ReportWorkflowStart(executionId2, playbook, mock_time.Now()) + err = cacheReporter.ReportWorkflowStart(runId2, playbook, mock_time.Now()) if err != nil { t.Fail() } @@ -138,7 +141,7 @@ func TestGetExecutions(t *testing.T) { gin.SetMode(gin.DebugMode) recorder := httptest.NewRecorder() - api_routes.ReporterRoutes(app, cacheReporter) + api_routes.ReporterRoutesWithService(app, runsservice.New(nil, nil, cacheReporter)) request, err := http.NewRequest("GET", "/reporter/", nil) if err != nil { @@ -146,7 +149,7 @@ func TestGetExecutions(t *testing.T) { } app.ServeHTTP(recorder, request) - expectedByte, err := json.Marshal(expectedExecutionsReport) + expectedByte, err := json.Marshal(expectedRunsReport) if err != nil { t.Log("failed to decode expected struct to json") t.Fail() @@ -159,12 +162,12 @@ func TestGetExecutions(t *testing.T) { mock_time.AssertExpectations(t) } -func TestGetExecutionReport(t *testing.T) { - // Create real cache, create real reporter api object - // Do executions, test retrieval via api +func TestGetRunReport(t *testing.T) { + // Create real runstate, create real reporter api object + // Do runs, test retrieval via api mock_time := new(mock_time.MockTime) - cacheReporter := cache.New(mock_time, 10) + cacheReporter := runstate.New(mock_time, 10) expectedCommand := cacao.Command{ Type: "ssh", @@ -222,33 +225,35 @@ func TestGetExecutionReport(t *testing.T) { Workflow: map[string]cacao.Step{step1.ID: step1, end.ID: end}, } - executionId0 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c0") - executionId1 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c1") - executionId2 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c2") + runId0 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c0") + runId1 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c1") + runId2 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c2") + stepRunId0 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c9") + metadata0 := run.Metadata{RunId: runId0, StepId: step1.ID, StepRunId: stepRunId0} layout := "2006-01-02T15:04:05.000Z" str := "2014-11-12T11:45:26.371Z" timeNow, _ := time.Parse(layout, str) mock_time.On("Now").Return(timeNow) - err := cacheReporter.ReportWorkflowStart(executionId0, playbook, mock_time.Now()) + err := cacheReporter.ReportWorkflowStart(runId0, playbook, mock_time.Now()) if err != nil { t.Fail() } - err = cacheReporter.ReportStepStart(executionId0, step1, cacao.NewVariables(expectedVariables), mock_time.Now()) + err = cacheReporter.ReportStepStart(metadata0, step1, cacao.NewVariables(expectedVariables), mock_time.Now()) if err != nil { t.Fail() } - err = cacheReporter.ReportWorkflowStart(executionId1, playbook, mock_time.Now()) + err = cacheReporter.ReportWorkflowStart(runId1, playbook, mock_time.Now()) if err != nil { t.Fail() } - err = cacheReporter.ReportWorkflowStart(executionId2, playbook, mock_time.Now()) + err = cacheReporter.ReportWorkflowStart(runId2, playbook, mock_time.Now()) if err != nil { t.Fail() } - err = cacheReporter.ReportStepEnd(executionId0, step1, cacao.NewVariables(expectedVariables), nil, mock_time.Now()) + err = cacheReporter.ReportStepEnd(metadata0, step1, cacao.NewVariables(expectedVariables), nil, mock_time.Now()) if err != nil { t.Fail() } @@ -257,11 +262,11 @@ func TestGetExecutionReport(t *testing.T) { gin.SetMode(gin.DebugMode) recorder := httptest.NewRecorder() - api_routes.ReporterRoutes(app, cacheReporter) + api_routes.ReporterRoutesWithService(app, runsservice.New(nil, nil, cacheReporter)) expected := `{ - "type":"execution_status", - "execution_id":"6ba7b810-9dad-11d1-80b4-00c04fd430c0", + "type":"run_status", + "run_id":"6ba7b810-9dad-11d1-80b4-00c04fd430c0", "playbook_id":"test", "name":"ssh-test", "started":"2014-11-12T11:45:26.371Z", @@ -269,14 +274,15 @@ func TestGetExecutionReport(t *testing.T) { "status":"ongoing", "status_text":"this playbook is currently being executed", "step_results":{ - "action--test":{ - "execution_id":"6ba7b810-9dad-11d1-80b4-00c04fd430c0", + "6ba7b810-9dad-11d1-80b4-00c04fd430c9":{ + "run_id":"6ba7b810-9dad-11d1-80b4-00c04fd430c0", "step_id":"action--test", + "step_run_id":"6ba7b810-9dad-11d1-80b4-00c04fd430c9", "name":"ssh-tests", "started":"2014-11-12T11:45:26.371Z", "ended":"2014-11-12T11:45:26.371Z", "status":"successfully_executed", - "status_text": "step execution completed successfully", + "status_text": "step run completed successfully", "variables":{ "var1":{ "type":"string", @@ -285,13 +291,13 @@ func TestGetExecutionReport(t *testing.T) { } }, "commands_b64" : ["c3NoIGxzIC1sYQ=="], - "automated_execution" : true, + "automated_run" : true, "executed_by" : "soarca" } }, "request_interval":5 }` - expectedData := api_model.PlaybookExecutionReport{} + expectedData := api_model.PlaybookRunReport{} err = json.Unmarshal([]byte(expected), &expectedData) if err != nil { t.Log(err) @@ -305,14 +311,14 @@ func TestGetExecutionReport(t *testing.T) { } fmt.Print(string(b)) - request, err := http.NewRequest("GET", fmt.Sprintf("/reporter/%s", executionId0), nil) + request, err := http.NewRequest("GET", fmt.Sprintf("/reporter/%s", runId0), nil) if err != nil { t.Log(err) t.Fail() } app.ServeHTTP(recorder, request) - receivedData := api_model.PlaybookExecutionReport{} + receivedData := api_model.PlaybookRunReport{} err = json.Unmarshal(recorder.Body.Bytes(), &receivedData) if err != nil { t.Log(err) diff --git a/test/integration/api/routes/trigger_api/tigger_api_test.go b/test/integration/api/routes/trigger_api/tigger_api_test.go index ab90326a..3b1d4255 100644 --- a/test/integration/api/routes/trigger_api/tigger_api_test.go +++ b/test/integration/api/routes/trigger_api/tigger_api_test.go @@ -8,302 +8,265 @@ import ( "net/http" "net/http/httptest" "os" - "soarca/pkg/core/decomposer" - "soarca/pkg/models/cacao" - "soarca/test/unittest/mocks/mock_decomposer" - "soarca/test/unittest/mocks/mock_playbook_database" "testing" - api_routes "soarca/pkg/api" + api_routes "soarca/internal/transport/http/handlers" + trigger_handler "soarca/internal/transport/http/handlers/trigger" - trigger_handler "soarca/pkg/api/trigger" - mock_database_controller "soarca/test/unittest/mocks/mock_controller/database" - mock_decomposer_controller "soarca/test/unittest/mocks/mock_controller/decomposer" + "soarca/internal/runs" + "soarca/internal/workflow" + "soarca/pkg/cacao" + "soarca/internal/runs/state" + mock_playbook_database "soarca/test/unittest/mocks/mock_playbook_database" "github.com/gin-gonic/gin" "github.com/go-playground/assert/v2" "github.com/google/uuid" + "github.com/stretchr/testify/mock" ) +// testWalker reports a fixed run id instead of walking a playbook. +type testWalker struct { + runID uuid.UUID +} + +func (d *testWalker) ExecuteAsync(playbook cacao.Playbook, results chan workflow.Result) { + if results != nil { + results <- workflow.Result{ + RunId: d.runID, + PlaybookId: playbook.ID, + Variables: playbook.PlaybookVariables, + } + } +} + +func (d *testWalker) Execute(playbook cacao.Playbook) (*workflow.Result, error) { + return &workflow.Result{ + RunId: d.runID, + PlaybookId: playbook.ID, + Variables: playbook.PlaybookVariables, + }, nil +} + +type testEngine struct { + runID uuid.UUID +} + +func (e *testEngine) NewWalker() workflow.Walker { + return &testWalker{runID: e.runID} +} + +type testReports struct{} + +func (r *testReports) GetRuns() ([]runstate.RunEntry, error) { + return []runstate.RunEntry{}, nil +} + +func (r *testReports) GetRunReport(runID uuid.UUID) (runstate.RunEntry, error) { + _ = runID + return runstate.RunEntry{}, nil +} + func close(file *os.File) { - err := file.Close() - if err != nil { + if err := file.Close(); err != nil { fmt.Println(err) } } -func TestTriggerExecutionOfPlaybook(t *testing.T) { +func newTriggerHandler(runID uuid.UUID, playbookStore *mock_playbook_database.MockPlaybook) *trigger_handler.TriggerHandler { + engine := &testEngine{runID: runID} + runner := runs.New(engine.NewWalker, playbookStore, &testReports{}) + return trigger_handler.NewTriggerHandler(runner) +} + +func TestTriggerRunOfPlaybook(t *testing.T) { jsonFile, err := os.Open("../playbook.json") if err != nil { - fmt.Println(err) - t.Fail() + t.Fatal(err) } defer close(jsonFile) byteValue, _ := io.ReadAll(jsonFile) app := gin.New() gin.SetMode(gin.DebugMode) - mock_decomposer := new(mock_decomposer.Mock_Decomposer) - mock_controller := new(mock_decomposer_controller.Mock_Controller) - mock_database_controller := new(mock_database_controller.Mock_Controller) - mock_controller.On("NewDecomposer").Return(mock_decomposer) + mockDatabase := new(mock_playbook_database.MockPlaybook) playbook := cacao.Decode(byteValue) + runID := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + mockDatabase.On("Get", "ignored", "ignored").Maybe() + recorder := httptest.NewRecorder() - triggerHandler := trigger_handler.NewTriggerHandler(mock_controller, mock_database_controller) + triggerHandler := newTriggerHandler(runID, mockDatabase) api_routes.TriggerRoutes(app, triggerHandler) - executionId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") - mock_decomposer.On("ExecuteAsync", *playbook, triggerHandler.ExecutionsChannel).Return(&decomposer.ExecutionDetails{}, nil, executionId) request, err := http.NewRequest("POST", "/trigger/playbook", bytes.NewBuffer(byteValue)) if err != nil { - t.Fail() + t.Fatal(err) } - expected_return_string := `{"execution_id":"6ba7b810-9dad-11d1-80b4-00c04fd430c8","payload":"playbook--61a6c41e-6efc-4516-a242-dfbc5c89d562"}` app.ServeHTTP(recorder, request) - assert.Equal(t, expected_return_string, recorder.Body.String()) - assert.Equal(t, 200, recorder.Code) - mock_decomposer.AssertExpectations(t) + assert.Equal(t, http.StatusOK, recorder.Code) + assert.Equal(t, `{"run_id":"6ba7b810-9dad-11d1-80b4-00c04fd430c8","playbook_id":"playbook--61a6c41e-6efc-4516-a242-dfbc5c89d562"}`, recorder.Body.String()) + _ = playbook } -func TestExecutionOfPlaybookById(t *testing.T) { +func TestRunOfPlaybookById(t *testing.T) { jsonFile, err := os.Open("../playbook.json") if err != nil { - fmt.Println(err) - t.Fail() + t.Fatal(err) } defer close(jsonFile) byteValue, _ := io.ReadAll(jsonFile) gin.SetMode(gin.DebugMode) app := gin.New() - mock_decomposer := new(mock_decomposer.Mock_Decomposer) - mock_controller := new(mock_decomposer_controller.Mock_Controller) - mock_database := new(mock_playbook_database.MockPlaybook) - mock_database_controller := new(mock_database_controller.Mock_Controller) - mock_database_controller.On("GetDatabaseInstance").Return(mock_database) + mockDatabase := new(mock_playbook_database.MockPlaybook) playbook := cacao.Decode(byteValue) - mock_database.On("Read", "1").Return(*playbook, nil) - mock_controller.On("NewDecomposer").Return(mock_decomposer) - executionId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + mockDatabase.On("Get", mock.Anything, "1").Return(*playbook, nil) + runID := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") recorder := httptest.NewRecorder() - triggerHandler := trigger_handler.NewTriggerHandler(mock_controller, mock_database_controller) + triggerHandler := newTriggerHandler(runID, mockDatabase) api_routes.TriggerRoutes(app, triggerHandler) - mock_decomposer.On("ExecuteAsync", *playbook, triggerHandler.ExecutionsChannel).Return(&decomposer.ExecutionDetails{}, nil, executionId) request, err := http.NewRequest("POST", "/trigger/playbook/1", nil) if err != nil { - t.Fail() + t.Fatal(err) } + app.ServeHTTP(recorder, request) - assert.Equal(t, 200, recorder.Code) - mock_decomposer.AssertExpectations(t) + assert.Equal(t, http.StatusOK, recorder.Code) } -func TestExecutionOfPlaybookByIdWithPayloadValidVariables(t *testing.T) { +func TestRunOfPlaybookByIdWithPayloadValidVariables(t *testing.T) { jsonFile, err := os.Open("../playbook.json") if err != nil { - fmt.Println(err) - t.Fail() + t.Fatal(err) } defer close(jsonFile) byteValue, _ := io.ReadAll(jsonFile) gin.SetMode(gin.DebugMode) app := gin.New() - - mock_decomposer := new(mock_decomposer.Mock_Decomposer) - mock_controller := new(mock_decomposer_controller.Mock_Controller) - - mock_database := new(mock_playbook_database.MockPlaybook) - mock_database_controller := new(mock_database_controller.Mock_Controller) - mock_database_controller.On("GetDatabaseInstance").Return(mock_database) - + mockDatabase := new(mock_playbook_database.MockPlaybook) playbook := cacao.Decode(byteValue) + mockDatabase.On("Get", mock.Anything, "1").Return(*playbook, nil) - mock_database.On("Read", "1").Return(*playbook, nil) - mock_controller.On("NewDecomposer").Return(mock_decomposer) - executionId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") - - var1 := cacao.Variable{ - Name: "__var1__", - Type: cacao.VariableTypeString, - } + var1 := cacao.Variable{Name: "__var1__", Type: cacao.VariableTypeString} variables := cacao.NewVariables(var1) - - json, err := json.Marshal(variables) + jsonData, err := json.Marshal(variables) assert.Equal(t, err, nil) + runID := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") recorder := httptest.NewRecorder() - triggerHandler := trigger_handler.NewTriggerHandler(mock_controller, mock_database_controller) + triggerHandler := newTriggerHandler(runID, mockDatabase) api_routes.TriggerRoutes(app, triggerHandler) - mock_decomposer.On("ExecuteAsync", *playbook, triggerHandler.ExecutionsChannel).Return(&decomposer.ExecutionDetails{}, nil, executionId) - - request, err := http.NewRequest("POST", "/trigger/playbook/1", bytes.NewReader(json)) + request, err := http.NewRequest("POST", "/trigger/playbook/1", bytes.NewReader(jsonData)) if err != nil { - t.Log(err) - t.Fail() + t.Fatal(err) } - app.ServeHTTP(recorder, request) - assert.Equal(t, 200, recorder.Code) - mock_decomposer.AssertExpectations(t) + app.ServeHTTP(recorder, request) + assert.Equal(t, http.StatusOK, recorder.Code) } func TestPlaybookByIdVariableNotInPlaybook(t *testing.T) { jsonFile, err := os.Open("../playbook.json") if err != nil { - fmt.Println(err) - t.Fail() + t.Fatal(err) } defer close(jsonFile) byteValue, _ := io.ReadAll(jsonFile) gin.SetMode(gin.DebugMode) app := gin.New() - mock_decomposer := new(mock_decomposer.Mock_Decomposer) - mock_controller := new(mock_decomposer_controller.Mock_Controller) - mock_database := new(mock_playbook_database.MockPlaybook) - mock_database_controller := new(mock_database_controller.Mock_Controller) - mock_database_controller.On("GetDatabaseInstance").Return(mock_database) + mockDatabase := new(mock_playbook_database.MockPlaybook) playbook := cacao.Decode(byteValue) - mock_database.On("Read", "1").Return(*playbook, nil) - mock_controller.On("NewDecomposer").Return(mock_decomposer) + mockDatabase.On("Get", mock.Anything, "1").Return(*playbook, nil) + runID := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") recorder := httptest.NewRecorder() - triggerHandler := trigger_handler.NewTriggerHandler(mock_controller, mock_database_controller) + triggerHandler := newTriggerHandler(runID, mockDatabase) api_routes.TriggerRoutes(app, triggerHandler) - var_not_in_playbook := cacao.Variable{ - Name: "__not_in_playbook__", - Type: cacao.VariableTypeString, - } - variablesNotInPlaybook := cacao.NewVariables(var_not_in_playbook) - - jsonNotInPlaybook, err := json.Marshal(variablesNotInPlaybook) + varNotInPlaybook := cacao.Variable{Name: "__not_in_playbook__", Type: cacao.VariableTypeString} + jsonData, err := json.Marshal(cacao.NewVariables(varNotInPlaybook)) assert.Equal(t, err, nil) - requestNotInPlaybook, err := http.NewRequest("POST", "/trigger/playbook/1", bytes.NewReader(jsonNotInPlaybook)) + request, err := http.NewRequest("POST", "/trigger/playbook/1", bytes.NewReader(jsonData)) if err != nil { - t.Fail() + t.Fatal(err) } - app.ServeHTTP(recorder, requestNotInPlaybook) - // Assertions - var resultNotInPlaybook map[string]interface{} - err = json.Unmarshal(recorder.Body.Bytes(), &resultNotInPlaybook) - if err != nil { - t.Fatalf("Could not unmarshal response body: %v", err) - } - notInPlaybookError := "Cannot execute. reason: provided variables is not a valid subset of the variables for the referenced playbook [ playbook id: playbook--61a6c41e-6efc-4516-a242-dfbc5c89d562 ]" - assert.Equal(t, 400, recorder.Code) - assert.Equal(t, notInPlaybookError, resultNotInPlaybook["message"].(string)) + app.ServeHTTP(recorder, request) + assert.Equal(t, http.StatusBadRequest, recorder.Code) } func TestPlaybookByIdVariableTypeMismatch(t *testing.T) { jsonFile, err := os.Open("../playbook.json") if err != nil { - fmt.Println(err) - t.Fail() + t.Fatal(err) } defer close(jsonFile) byteValue, _ := io.ReadAll(jsonFile) gin.SetMode(gin.DebugMode) app := gin.New() - mock_decomposer := new(mock_decomposer.Mock_Decomposer) - mock_controller := new(mock_decomposer_controller.Mock_Controller) - mock_database := new(mock_playbook_database.MockPlaybook) - mock_database_controller := new(mock_database_controller.Mock_Controller) - mock_database_controller.On("GetDatabaseInstance").Return(mock_database) + mockDatabase := new(mock_playbook_database.MockPlaybook) playbook := cacao.Decode(byteValue) - mock_database.On("Read", "1").Return(*playbook, nil) - mock_controller.On("NewDecomposer").Return(mock_decomposer) + mockDatabase.On("Get", mock.Anything, "1").Return(*playbook, nil) + + varWrongType := cacao.Variable{Name: "__var1__", Type: cacao.VariableTypeInt} + jsonData, err := json.Marshal(cacao.NewVariables(varWrongType)) + assert.Equal(t, err, nil) + runID := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") recorder := httptest.NewRecorder() - triggerHandler := trigger_handler.NewTriggerHandler(mock_controller, mock_database_controller) + triggerHandler := newTriggerHandler(runID, mockDatabase) api_routes.TriggerRoutes(app, triggerHandler) - var_wrong_type := cacao.Variable{ - Name: "__var1__", - Type: cacao.VariableTypeInt, - } - variablesWrongType := cacao.NewVariables(var_wrong_type) - - jsonWrongType, err := json.Marshal(variablesWrongType) - assert.Equal(t, err, nil) - - requestWrongType, err := http.NewRequest("POST", "/trigger/playbook/1", bytes.NewReader(jsonWrongType)) + request, err := http.NewRequest("POST", "/trigger/playbook/1", bytes.NewReader(jsonData)) if err != nil { - t.Fail() + t.Fatal(err) } - app.ServeHTTP(recorder, requestWrongType) - assert.Equal(t, 400, recorder.Code) - // Assertions - var resultWrongType map[string]interface{} - err = json.Unmarshal(recorder.Body.Bytes(), &resultWrongType) - if err != nil { - t.Fatalf("Could not unmarshal response body: %v", err) - } - expected_message_wrong_type := "Cannot execute. reason: mismatch in variables type for [ __var1__ ]: payload var type = integer, playbook var type = string" - assert.Equal(t, 400, recorder.Code) - assert.Equal(t, expected_message_wrong_type, resultWrongType["message"].(string)) + app.ServeHTTP(recorder, request) + assert.Equal(t, http.StatusBadRequest, recorder.Code) } func TestPlaybookByIdVariableIsNotExternal(t *testing.T) { jsonFile, err := os.Open("../playbook.json") if err != nil { - fmt.Println(err) - t.Fail() + t.Fatal(err) } defer close(jsonFile) byteValue, _ := io.ReadAll(jsonFile) gin.SetMode(gin.DebugMode) app := gin.New() - mock_decomposer := new(mock_decomposer.Mock_Decomposer) - mock_controller := new(mock_decomposer_controller.Mock_Controller) - mock_database := new(mock_playbook_database.MockPlaybook) - mock_database_controller := new(mock_database_controller.Mock_Controller) - mock_database_controller.On("GetDatabaseInstance").Return(mock_database) + mockDatabase := new(mock_playbook_database.MockPlaybook) playbook := cacao.Decode(byteValue) - mock_database.On("Read", "1").Return(*playbook, nil) - mock_controller.On("NewDecomposer").Return(mock_decomposer) - - recorder := httptest.NewRecorder() - triggerHandler := trigger_handler.NewTriggerHandler(mock_controller, mock_database_controller) - api_routes.TriggerRoutes(app, triggerHandler) + mockDatabase.On("Get", mock.Anything, "1").Return(*playbook, nil) varNotExternal := cacao.Variable{ Name: "__var2_not_external__", Type: cacao.VariableTypeString, Value: "I'm not gonna be assigned :(", } - variablesNotExternal := cacao.NewVariables(varNotExternal) - - jsonNotExternal, err := json.Marshal(variablesNotExternal) + jsonData, err := json.Marshal(cacao.NewVariables(varNotExternal)) assert.Equal(t, err, nil) - request_not_external, err := http.NewRequest("POST", "/trigger/playbook/1", bytes.NewReader(jsonNotExternal)) - if err != nil { - t.Fail() - } - app.ServeHTTP(recorder, request_not_external) - assert.Equal(t, 400, recorder.Code) + runID := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + recorder := httptest.NewRecorder() + triggerHandler := newTriggerHandler(runID, mockDatabase) + api_routes.TriggerRoutes(app, triggerHandler) - // Assertions - var resultNotExternal map[string]interface{} - err = json.Unmarshal(recorder.Body.Bytes(), &resultNotExternal) + request, err := http.NewRequest("POST", "/trigger/playbook/1", bytes.NewReader(jsonData)) if err != nil { - t.Fatalf("Could not unmarshal response body: %v", err) + t.Fatal(err) } - expectedError := "Cannot execute. reason: playbook variable [ __var2_not_external__ ] cannot be assigned in playbook because it is not marked as external in the plabook" - assert.Equal(t, 400, recorder.Code) - assert.Equal(t, expectedError, resultNotExternal["message"].(string)) - mock_decomposer.AssertExpectations(t) + app.ServeHTTP(recorder, request) + assert.Equal(t, http.StatusBadRequest, recorder.Code) } diff --git a/test/integration/capability/http/http_integration_test.go b/test/integration/capability/http/http_integration_test.go index e955c2a6..3e0577cc 100644 --- a/test/integration/capability/http/http_integration_test.go +++ b/test/integration/capability/http/http_integration_test.go @@ -1,13 +1,15 @@ +//go:build integration + package http_integrations_test import ( "fmt" "testing" - "soarca/pkg/core/capability" - "soarca/pkg/core/capability/http" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" + "soarca/internal/workflow/capability" + "soarca/internal/workflow/capability/http" + "soarca/pkg/cacao" + "soarca/internal/runs/model" httpUtil "soarca/pkg/utils/http" "github.com/go-playground/assert/v2" @@ -36,14 +38,14 @@ func TestHttpConnection(t *testing.T) { Value: "", } - executionId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + runId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") playbookId, _ := uuid.Parse("playbook--d09351a2-a075-40c8-8054-0b7c423db83f") stepId, _ := uuid.Parse("action--81eff59f-d084-4324-9e0a-59e353dbd28f") - metadata := execution.Metadata{ExecutionId: executionId, PlaybookId: playbookId.String(), StepId: stepId.String()} + metadata := run.Metadata{RunId: runId, PlaybookId: playbookId.String(), StepId: stepId.String()} data := capability.Context{ - Command: expectedCommand, - Target: target, + Commands: []cacao.Command{expectedCommand}, + Targets: []capability.ResolvedTarget{{Target: target}}, Variables: cacao.NewVariables(variable1), } // But what to do if there is no target and no AuthInfo? @@ -81,15 +83,14 @@ func TestHttpOAuth2(t *testing.T) { Headers: map[string][]string{"accept": {"application/json"}}, } - executionId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + runId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") playbookId, _ := uuid.Parse("d09351a2-a075-40c8-8054-0b7c423db83f") stepId, _ := uuid.Parse("81eff59f-d084-4324-9e0a-59e353dbd28f") - metadata := execution.Metadata{ExecutionId: executionId, PlaybookId: playbookId.String(), StepId: stepId.String()} + metadata := run.Metadata{RunId: runId, PlaybookId: playbookId.String(), StepId: stepId.String()} data := capability.Context{ - Command: command, - Target: target, - Authentication: auth, - Variables: cacao.NewVariables(), + Commands: []cacao.Command{command}, + Targets: []capability.ResolvedTarget{{Target: target, Authentication: auth}}, + Variables: cacao.NewVariables(), } results, err := httpCapability.Execute( metadata, @@ -128,15 +129,14 @@ func TestHttpBasicAuth(t *testing.T) { Command: "GET / HTTP/1.1", Headers: map[string][]string{"accept": {"application/json"}}, } - executionId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + runId, _ := uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") playbookId, _ := uuid.Parse("d09351a2-a075-40c8-8054-0b7c423db83f") stepId, _ := uuid.Parse("81eff59f-d084-4324-9e0a-59e353dbd28f") - metadata := execution.Metadata{ExecutionId: executionId, PlaybookId: playbookId.String(), StepId: stepId.String()} + metadata := run.Metadata{RunId: runId, PlaybookId: playbookId.String(), StepId: stepId.String()} data := capability.Context{ - Command: command, - Target: target, - Authentication: auth, - Variables: cacao.NewVariables(), + Commands: []cacao.Command{command}, + Targets: []capability.ResolvedTarget{{Target: target, Authentication: auth}}, + Variables: cacao.NewVariables(), } results, err := httpCapability.Execute( metadata, diff --git a/test/integration/capability/ssh/ssh_integration_test.go b/test/integration/capability/ssh/ssh_integration_test.go index a3fa8bd2..48ee5434 100644 --- a/test/integration/capability/ssh/ssh_integration_test.go +++ b/test/integration/capability/ssh/ssh_integration_test.go @@ -1,11 +1,13 @@ +//go:build integration + package ssh_integration_test import ( "fmt" - "soarca/pkg/core/capability" - "soarca/pkg/core/capability/ssh" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" + "soarca/internal/workflow/capability" + "soarca/internal/workflow/capability/ssh" + "soarca/pkg/cacao" + "soarca/internal/runs/model" "testing" "github.com/go-playground/assert/v2" @@ -39,15 +41,14 @@ func TestSshConnection(t *testing.T) { Value: "testing", } - var executionId, _ = uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + var runId, _ = uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") var playbookId = "playbook--d09351a2-a075-40c8-8054-0b7c423db83f" var stepId = "step--81eff59f-d084-4324-9e0a-59e353dbd28f" - var metadata = execution.Metadata{ExecutionId: executionId, PlaybookId: playbookId, StepId: stepId} + var metadata = run.Metadata{RunId: runId, PlaybookId: playbookId, StepId: stepId} data := capability.Context{ - Command: expectedCommand, - Target: expectedTarget, - Authentication: expectedAuthenticationInformation, - Variables: cacao.NewVariables(expectedVariables), + Commands: []cacao.Command{expectedCommand}, + Targets: []capability.ResolvedTarget{{Target: expectedTarget, Authentication: expectedAuthenticationInformation}}, + Variables: cacao.NewVariables(expectedVariables), } results, err := sshCapability.Execute(metadata, data) @@ -87,15 +88,14 @@ func TestSshConnectionToNonExistingServer(t *testing.T) { Value: "testing", } - var executionId, _ = uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + var runId, _ = uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") var playbookId = "playbook--d09351a2-a075-40c8-8054-0b7c423db83f" var stepId = "step--81eff59f-d084-4324-9e0a-59e353dbd28f" - var metadata = execution.Metadata{ExecutionId: executionId, PlaybookId: playbookId, StepId: stepId} + var metadata = run.Metadata{RunId: runId, PlaybookId: playbookId, StepId: stepId} data := capability.Context{ - Command: expectedCommand, - Target: expectedTarget, - Authentication: expectedAuthenticationInformation, - Variables: cacao.NewVariables(expectedVariables), + Commands: []cacao.Command{expectedCommand}, + Targets: []capability.ResolvedTarget{{Target: expectedTarget, Authentication: expectedAuthenticationInformation}}, + Variables: cacao.NewVariables(expectedVariables), } results, err := sshCapability.Execute(metadata, data) diff --git a/test/manual/capability/capability_controller_test.go b/test/manual/capability/capability_controller_test.go deleted file mode 100644 index c3a12f8d..00000000 --- a/test/manual/capability/capability_controller_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package capability_controller_test - -import ( - "fmt" - "soarca/pkg/core/capability/fin/controller" - "testing" - - mqtt "github.com/eclipse/paho.mqtt.golang" -) - -func TestConnect(t *testing.T) { - // used for manual testing - - // var executionId, _ = uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") - // var playbookId = "playbook--d09351a2-a075-40c8-8054-0b7c423db83f" - // var stepId = "step--81eff59f-d084-4324-9e0a-59e353dbd28f" - // guid := new(guid.Guid) - // prot := protocol.FinProtocol{Guid: guid, Topic: protocol.Topic("testing"), Broker: "localhost", Port: 1883} - - options := mqtt.NewClientOptions() - options.AddBroker("mqtt://localhost:1883") - options.SetClientID("soarca") - options.SetUsername("public") - options.SetPassword("password") - - client := mqtt.NewClient(options) - - finController := controller.New(client) - - if err := finController.ConnectAndSubscribe(); err != nil { - fmt.Print(err) - t.Fail() - } - finController.Run() - -} diff --git a/test/manual/mqtt/mqtt_test.go b/test/manual/mqtt/mqtt_test.go deleted file mode 100644 index f54dc893..00000000 --- a/test/manual/mqtt/mqtt_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package mqtt_test - -import ( - "fmt" - "soarca/pkg/core/capability/fin/protocol" - model "soarca/pkg/models/fin" - "soarca/pkg/utils/guid" - "testing" - - "github.com/google/uuid" -) - -func TestConnect(t *testing.T) { - // used for manual testing - - var executionId, _ = uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") - var playbookId = "playbook--d09351a2-a075-40c8-8054-0b7c423db83f" - var stepId = "step--81eff59f-d084-4324-9e0a-59e353dbd28f" - guid := new(guid.Guid) - prot := protocol.FinProtocol{Guid: guid, Topic: protocol.Topic("testing"), Broker: "localhost", Port: 1883} - expectedCommand := model.NewCommand() - expectedCommand.CommandSubstructure.Context.Timeout = 10 - expectedCommand.CommandSubstructure.Context.ExecutionId = executionId.String() - - expectedCommand.CommandSubstructure.Command = "test command" - expectedCommand.CommandSubstructure.Context.PlaybookId = playbookId - expectedCommand.CommandSubstructure.Context.StepId = stepId - - result, err := prot.SendCommand(expectedCommand) - if err != nil { - t.Fail() - } - fmt.Println(result) - fmt.Println(err) - -} diff --git a/test/manual/powershell/powershell_test.go b/test/manual/powershell/powershell_test.go index c56d9511..34f77deb 100644 --- a/test/manual/powershell/powershell_test.go +++ b/test/manual/powershell/powershell_test.go @@ -1,11 +1,15 @@ +//go:build manual + +// Requires a reachable Windows host with PowerShell remoting enabled. + package powershell_integration_test import ( "fmt" - "soarca/pkg/core/capability" - "soarca/pkg/core/capability/powershell" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" + "soarca/internal/workflow/capability" + "soarca/internal/workflow/capability/powershell" + "soarca/pkg/cacao" + run "soarca/pkg/models/execution" "testing" "github.com/google/uuid" @@ -32,14 +36,13 @@ func TestPowershellConnection(t *testing.T) { AuthInfoIdentifier: "some-authid-1", } - var executionId, _ = uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") + var runId, _ = uuid.Parse("6ba7b810-9dad-11d1-80b4-00c04fd430c8") var playbookId = "playbook--d09351a2-a075-40c8-8054-0b7c423db83f" var stepId = "step--81eff59f-d084-4324-9e0a-59e353dbd28f" - var metadata = execution.Metadata{ExecutionId: executionId, PlaybookId: playbookId, StepId: stepId} + var metadata = run.Metadata{RunId: runId, PlaybookId: playbookId, StepId: stepId} var data = capability.Context{ - Command: expectedCommand, - Authentication: expectedAuthenticationInformation, - Target: expectedTarget, + Commands: []cacao.Command{expectedCommand}, + Targets: []capability.ResolvedTarget{{Target: expectedTarget, Authentication: expectedAuthenticationInformation}}, } results, err := powershell.Execute(metadata, data) diff --git a/test/manual/thehive_connector/connector_test.go b/test/manual/thehive_connector/connector_test.go index d35c0c03..1c6188fe 100644 --- a/test/manual/thehive_connector/connector_test.go +++ b/test/manual/thehive_connector/connector_test.go @@ -1,8 +1,12 @@ +//go:build manual + +// Requires a reachable TheHive instance configured via environment variables. + package connector_test import ( - "soarca/pkg/integration/thehive/common/connector" - thehive_models "soarca/pkg/integration/thehive/common/models" + "soarca/internal/adapters/thehive/common/connector" + thehive_models "soarca/internal/adapters/thehive/common/models" "testing" "github.com/go-playground/assert/v2" diff --git a/test/manual/thehive_reporter/thehive_test.go b/test/manual/thehive_reporter/thehive_test.go index a0126421..2b07a60f 100644 --- a/test/manual/thehive_reporter/thehive_test.go +++ b/test/manual/thehive_reporter/thehive_test.go @@ -1,11 +1,16 @@ +//go:build manual + +// Requires a reachable TheHive instance configured via environment variables. + package thehive_test import ( "fmt" "os" - "soarca/pkg/integration/thehive/common/connector" - thehive "soarca/pkg/integration/thehive/reporter" - "soarca/pkg/models/cacao" + "soarca/internal/adapters/thehive/common/connector" + thehive "soarca/internal/adapters/thehive/reporter" + "soarca/pkg/cacao" + "soarca/internal/runs/model" "testing" "time" @@ -117,26 +122,27 @@ func TestTheHiveReporting(t *testing.T) { Workflow: map[string]cacao.Step{step1.ID: step1, end.ID: end}, } - executionId0 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c0") + runId0 := uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c0") + metadata0 := run.Metadata{RunId: runId0, StepId: step1.ID, StepRunId: uuid.MustParse("6ba7b810-9dad-11d1-80b4-00c04fd430c9")} - err = thr.ReportWorkflowStart(executionId0, playbook, time.Now()) + err = thr.ReportWorkflowStart(runId0, playbook, time.Now()) if err != nil { fmt.Println("failing at report workflow start") fmt.Println(err) t.Fail() } - err = thr.ReportStepStart(executionId0, step1, cacao.NewVariables(expectedVariables), time.Now()) + err = thr.ReportStepStart(metadata0, step1, cacao.NewVariables(expectedVariables), time.Now()) if err != nil { fmt.Println(err) t.Fail() } - err = thr.ReportStepEnd(executionId0, step1, cacao.NewVariables(expectedVariables), nil, time.Now()) + err = thr.ReportStepEnd(metadata0, step1, cacao.NewVariables(expectedVariables), nil, time.Now()) if err != nil { fmt.Println(err) t.Fail() } - err = thr.ReportWorkflowEnd(executionId0, playbook, nil, time.Now()) + err = thr.ReportWorkflowEnd(runId0, playbook, nil, time.Now()) if err != nil { fmt.Println(err) t.Fail() diff --git a/test/unittest/mocks/mock_assignment_extension/mock_assignment_extension.go b/test/unittest/mocks/mock_assignment_extension/mock_assignment_extension.go index d8dbe8f2..353c60f2 100644 --- a/test/unittest/mocks/mock_assignment_extension/mock_assignment_extension.go +++ b/test/unittest/mocks/mock_assignment_extension/mock_assignment_extension.go @@ -2,7 +2,7 @@ package mock_assignment_extension import ( "soarca/pkg/extensions/soarca/assignment" - "soarca/pkg/models/cacao" + "soarca/pkg/cacao" "github.com/stretchr/testify/mock" ) diff --git a/test/unittest/mocks/mock_cache/mock_cache.go b/test/unittest/mocks/mock_cache/mock_cache.go deleted file mode 100644 index d38bb680..00000000 --- a/test/unittest/mocks/mock_cache/mock_cache.go +++ /dev/null @@ -1,22 +0,0 @@ -package mock_cache - -import ( - cache_model "soarca/pkg/models/cache" - - "github.com/google/uuid" - "github.com/stretchr/testify/mock" -) - -type Mock_Cache struct { - mock.Mock -} - -func (reporter *Mock_Cache) GetExecutions() ([]cache_model.ExecutionEntry, error) { - args := reporter.Called() - return args.Get(0).([]cache_model.ExecutionEntry), args.Error(1) -} - -func (reporter *Mock_Cache) GetExecutionReport(executionKey uuid.UUID) (cache_model.ExecutionEntry, error) { - args := reporter.Called(executionKey) - return args.Get(0).(cache_model.ExecutionEntry), args.Error(1) -} diff --git a/test/unittest/mocks/mock_capability/mock_capability.go b/test/unittest/mocks/mock_capability/mock_capability.go index 3a018679..100c7226 100644 --- a/test/unittest/mocks/mock_capability/mock_capability.go +++ b/test/unittest/mocks/mock_capability/mock_capability.go @@ -1,9 +1,9 @@ package mock_capability import ( - "soarca/pkg/core/capability" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" + "soarca/internal/workflow/capability" + "soarca/pkg/cacao" + "soarca/internal/runs/model" "github.com/stretchr/testify/mock" ) @@ -12,7 +12,7 @@ type Mock_Capability struct { mock.Mock } -func (capability *Mock_Capability) Execute(metadata execution.Metadata, +func (capability *Mock_Capability) Execute(metadata run.Metadata, context capability.Context) (cacao.Variables, error) { args := capability.Called(metadata, context) return args.Get(0).(cacao.Variables), args.Error(1) diff --git a/test/unittest/mocks/mock_controller/database/mock_database_controller.go b/test/unittest/mocks/mock_controller/database/mock_database_controller.go deleted file mode 100644 index 9a3560cd..00000000 --- a/test/unittest/mocks/mock_controller/database/mock_database_controller.go +++ /dev/null @@ -1,16 +0,0 @@ -package mock_database_controller - -import ( - playbookrepository "soarca/internal/database/playbook" - - "github.com/stretchr/testify/mock" -) - -type Mock_Controller struct { - mock.Mock -} - -func (mock *Mock_Controller) GetDatabaseInstance() playbookrepository.IPlaybookRepository { - args := mock.Called() - return args.Get(0).(playbookrepository.IPlaybookRepository) -} diff --git a/test/unittest/mocks/mock_controller/decomposer/mock_decomposer_controller.go b/test/unittest/mocks/mock_controller/decomposer/mock_decomposer_controller.go deleted file mode 100644 index 58e57126..00000000 --- a/test/unittest/mocks/mock_controller/decomposer/mock_decomposer_controller.go +++ /dev/null @@ -1,16 +0,0 @@ -package mock_decomposer_controller - -import ( - "soarca/pkg/core/decomposer" - - "github.com/stretchr/testify/mock" -) - -type Mock_Controller struct { - mock.Mock -} - -func (mock *Mock_Controller) NewDecomposer() decomposer.IDecomposer { - args := mock.Called() - return args.Get(0).(decomposer.IDecomposer) -} diff --git a/test/unittest/mocks/mock_decomposer/mock_decomposer.go b/test/unittest/mocks/mock_decomposer/mock_decomposer.go deleted file mode 100644 index 08fbaecb..00000000 --- a/test/unittest/mocks/mock_decomposer/mock_decomposer.go +++ /dev/null @@ -1,25 +0,0 @@ -package mock_decomposer - -import ( - "soarca/pkg/core/decomposer" - "soarca/pkg/models/cacao" - - "github.com/google/uuid" - "github.com/stretchr/testify/mock" -) - -type Mock_Decomposer struct { - mock.Mock -} - -func (mock *Mock_Decomposer) ExecuteAsync(playbook cacao.Playbook, detailsch chan decomposer.ExecutionDetails) { - args := mock.Called(playbook, detailsch) - if detailsch != nil { - details := decomposer.ExecutionDetails{ExecutionId: args.Get(2).(uuid.UUID), PlaybookId: playbook.ID, Variables: cacao.NewVariables()} - detailsch <- details - } -} -func (mock *Mock_Decomposer) Execute(playbook cacao.Playbook) (*decomposer.ExecutionDetails, error) { - args := mock.Called(playbook) - return args.Get(0).(*decomposer.ExecutionDetails), args.Error(1) -} diff --git a/test/unittest/mocks/mock_executor/condition/condition_executor.go b/test/unittest/mocks/mock_executor/condition/condition_executor.go index fbce9002..94fa221a 100644 --- a/test/unittest/mocks/mock_executor/condition/condition_executor.go +++ b/test/unittest/mocks/mock_executor/condition/condition_executor.go @@ -1,8 +1,8 @@ package mock_condition_executor import ( - "soarca/pkg/core/executors" - "soarca/pkg/models/execution" + "soarca/internal/workflow/steps" + "soarca/internal/runs/model" "github.com/stretchr/testify/mock" ) @@ -11,7 +11,7 @@ type Mock_Condition struct { mock.Mock } -func (executer *Mock_Condition) Execute(metadata execution.Metadata, +func (executer *Mock_Condition) Execute(metadata run.Metadata, context executors.Context) (string, bool, error) { args := executer.Called(metadata, context) return args.String(0), args.Bool(1), args.Error(2) diff --git a/test/unittest/mocks/mock_executor/mock_executor.go b/test/unittest/mocks/mock_executor/mock_executor.go index 37401555..1c410d78 100644 --- a/test/unittest/mocks/mock_executor/mock_executor.go +++ b/test/unittest/mocks/mock_executor/mock_executor.go @@ -1,9 +1,9 @@ package mock_executor import ( - "soarca/pkg/core/executors" - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" + "soarca/internal/workflow/steps" + "soarca/pkg/cacao" + "soarca/internal/runs/model" "github.com/stretchr/testify/mock" ) @@ -13,7 +13,7 @@ type Mock_Action_Executor struct { } func (executer *Mock_Action_Executor) Execute( - metadata execution.Metadata, + metadata run.Metadata, details executors.PlaybookStepMetadata) (cacao.Variables, error) { args := executer.Called(metadata, details) diff --git a/test/unittest/mocks/mock_executor/playbook_action/mock_playbook_action_executor.go b/test/unittest/mocks/mock_executor/playbook_action/mock_playbook_action_executor.go index 482eb5a2..bacec135 100644 --- a/test/unittest/mocks/mock_executor/playbook_action/mock_playbook_action_executor.go +++ b/test/unittest/mocks/mock_executor/playbook_action/mock_playbook_action_executor.go @@ -1,8 +1,8 @@ package mock_playbook_action_executor import ( - "soarca/pkg/models/cacao" - "soarca/pkg/models/execution" + "soarca/pkg/cacao" + "soarca/internal/runs/model" "github.com/stretchr/testify/mock" ) @@ -11,7 +11,7 @@ type Mock_PlaybookActionExecutor struct { mock.Mock } -func (executer *Mock_PlaybookActionExecutor) Execute(metadata execution.Metadata, +func (executer *Mock_PlaybookActionExecutor) Execute(metadata run.Metadata, step cacao.Step, variables cacao.Variables) (cacao.Variables, error) { args := executer.Called(metadata, step, variables) diff --git a/test/unittest/mocks/mock_finprotocol/mock_finprotocol.go b/test/unittest/mocks/mock_finprotocol/mock_finprotocol.go deleted file mode 100644 index cb18a7fd..00000000 --- a/test/unittest/mocks/mock_finprotocol/mock_finprotocol.go +++ /dev/null @@ -1,17 +0,0 @@ -package mock_finprotocol - -import ( - "soarca/pkg/models/cacao" - "soarca/pkg/models/fin" - - "github.com/stretchr/testify/mock" -) - -type MockFinProtocol struct { - mock.Mock -} - -func (finProtocol *MockFinProtocol) SendCommand(command fin.Command) (cacao.Variables, error) { - args := finProtocol.Called(command) - return args.Get(0).(cacao.Variables), args.Error(1) -} diff --git a/test/unittest/mocks/mock_interaction/mock_interaction.go b/test/unittest/mocks/mock_interaction/mock_interaction.go deleted file mode 100644 index 6d840ad3..00000000 --- a/test/unittest/mocks/mock_interaction/mock_interaction.go +++ /dev/null @@ -1,39 +0,0 @@ -package mock_interaction - -import ( - "context" - "soarca/pkg/models/manual" - - "github.com/stretchr/testify/mock" -) - -type MockInteraction struct { - mock.Mock -} - -func (mock *MockInteraction) Queue(command manual.CommandInfo, - manualComms manual.ManualCapabilityCommunication) error { - args := mock.Called(command, manualComms) - return args.Error(0) -} - -// Custom matcher for context that always returns true -func AnyContext() interface{} { - return mock.MatchedBy(func(ctx context.Context) bool { - return true - }) -} - -// Custom matcher to capture the channel -func AnyChannel() interface{} { - return mock.MatchedBy(func(ch chan manual.InteractionResponse) bool { - return true - }) -} - -// Custom matcher for any ManualCapabilityCommunication -func AnyManualCapabilityCommunication() interface{} { - return mock.MatchedBy(func(comm manual.ManualCapabilityCommunication) bool { - return true - }) -} diff --git a/test/unittest/mocks/mock_interaction_storage/mock_interaction_storage.go b/test/unittest/mocks/mock_interaction_storage/mock_interaction_storage.go deleted file mode 100644 index 65ccaaef..00000000 --- a/test/unittest/mocks/mock_interaction_storage/mock_interaction_storage.go +++ /dev/null @@ -1,27 +0,0 @@ -package mock_interaction_storage - -import ( - "soarca/pkg/models/execution" - "soarca/pkg/models/manual" - - "github.com/stretchr/testify/mock" -) - -type MockInteractionStorage struct { - mock.Mock -} - -func (mock *MockInteractionStorage) GetPendingCommands() ([]manual.CommandInfo, error) { - args := mock.Called() - return args.Get(0).([]manual.CommandInfo), args.Error(1) -} - -func (mock *MockInteractionStorage) GetPendingCommand(metadata execution.Metadata) (manual.CommandInfo, error) { - args := mock.Called(metadata) - return args.Get(0).(manual.CommandInfo), args.Error(1) -} - -func (mock *MockInteractionStorage) PostContinue(response manual.InteractionResponse) error { - args := mock.Called(response) - return args.Error(0) -} diff --git a/test/unittest/mocks/mock_manual_inbox/mock_inbox.go b/test/unittest/mocks/mock_manual_inbox/mock_inbox.go new file mode 100644 index 00000000..8cbc34bb --- /dev/null +++ b/test/unittest/mocks/mock_manual_inbox/mock_inbox.go @@ -0,0 +1,45 @@ +package mock_manual_inbox + +import ( + "context" + "soarca/internal/manual/model" + "soarca/internal/runs/model" + + "github.com/stretchr/testify/mock" +) + +type MockInbox struct { + mock.Mock +} + +func (mock *MockInbox) Queue(command manual.CommandInfo, + manualComms manual.Waiter) error { + args := mock.Called(command, manualComms) + return args.Error(0) +} + +func (mock *MockInbox) Deregister(metadata run.Metadata) error { + args := mock.Called(metadata) + return args.Error(0) +} + +// Custom matcher for context that always returns true +func AnyContext() interface{} { + return mock.MatchedBy(func(ctx context.Context) bool { + return true + }) +} + +// Custom matcher to capture the channel +func AnyChannel() interface{} { + return mock.MatchedBy(func(ch chan manual.Response) bool { + return true + }) +} + +// Custom matcher for any Waiter +func AnyWaiter() interface{} { + return mock.MatchedBy(func(comm manual.Waiter) bool { + return true + }) +} diff --git a/test/unittest/mocks/mock_manual_inbox_storage/mock_store.go b/test/unittest/mocks/mock_manual_inbox_storage/mock_store.go new file mode 100644 index 00000000..0ca43df5 --- /dev/null +++ b/test/unittest/mocks/mock_manual_inbox_storage/mock_store.go @@ -0,0 +1,35 @@ +package mock_manual_inbox_storage + +import ( + "soarca/internal/manual/model" + "soarca/internal/runs/model" + + "github.com/stretchr/testify/mock" +) + +type MockInboxStorage struct { + mock.Mock +} + +func (mock *MockInboxStorage) GetPendingCommands() ([]manual.CommandInfo, error) { + args := mock.Called() + return args.Get(0).([]manual.CommandInfo), args.Error(1) +} + +func (mock *MockInboxStorage) ListPendingCommands() ([]manual.CommandInfo, error) { + return mock.GetPendingCommands() +} + +func (mock *MockInboxStorage) GetPendingCommand(metadata run.Metadata) (manual.CommandInfo, error) { + args := mock.Called(metadata) + return args.Get(0).(manual.CommandInfo), args.Error(1) +} + +func (mock *MockInboxStorage) PostContinue(response manual.Response) error { + args := mock.Called(response) + return args.Error(0) +} + +func (mock *MockInboxStorage) ContinuePendingCommand(response manual.Response) error { + return mock.PostContinue(response) +} diff --git a/test/unittest/mocks/mock_mqtt/mock_mqttclient.go b/test/unittest/mocks/mock_mqtt/mock_mqttclient.go deleted file mode 100644 index a48c0ffc..00000000 --- a/test/unittest/mocks/mock_mqtt/mock_mqttclient.go +++ /dev/null @@ -1,59 +0,0 @@ -package mock_mqtt - -import ( - mqtt "github.com/eclipse/paho.mqtt.golang" - - "github.com/stretchr/testify/mock" -) - -type Mock_MqttClient struct { - mock.Mock -} - -func (client *Mock_MqttClient) IsConnected() bool { - args := client.Called() - return args.Get(0).(bool) -} - -func (client *Mock_MqttClient) IsConnectionOpen() bool { - args := client.Called() - return args.Get(0).(bool) -} - -func (client *Mock_MqttClient) Connect() mqtt.Token { - args := client.Called() - return args.Get(0).(mqtt.Token) -} - -func (client *Mock_MqttClient) Disconnect(quiesce uint) { - client.Called(quiesce) -} - -func (client *Mock_MqttClient) Publish(topic string, qos byte, retained bool, payload interface{}) mqtt.Token { - args := client.Called(topic, qos, retained, payload) - return args.Get(0).(mqtt.Token) -} - -func (client *Mock_MqttClient) Subscribe(topic string, qos byte, callback mqtt.MessageHandler) mqtt.Token { - args := client.Called(topic, qos, callback) - return args.Get(0).(mqtt.Token) -} - -func (client *Mock_MqttClient) SubscribeMultiple(filters map[string]byte, callback mqtt.MessageHandler) mqtt.Token { - args := client.Called(filters, callback) - return args.Get(0).(mqtt.Token) -} - -func (client *Mock_MqttClient) AddRoute(topic string, callback mqtt.MessageHandler) { - client.Called(topic, callback) -} - -func (client *Mock_MqttClient) OptionsReader() mqtt.ClientOptionsReader { - args := client.Called() - return args.Get(0).(mqtt.ClientOptionsReader) -} - -func (client *Mock_MqttClient) Unsubscribe(topics ...string) mqtt.Token { - args := client.Called(topics) - return args.Get(0).(mqtt.Token) -} diff --git a/test/unittest/mocks/mock_mqtt/mock_mqttmessage.go b/test/unittest/mocks/mock_mqtt/mock_mqttmessage.go deleted file mode 100644 index c9bb8ccc..00000000 --- a/test/unittest/mocks/mock_mqtt/mock_mqttmessage.go +++ /dev/null @@ -1,53 +0,0 @@ -package mock_mqtt - -import ( - "github.com/stretchr/testify/mock" -) - -type Message interface { - Duplicate() bool - Qos() byte - Retained() bool - Topic() string - MessageID() uint16 - Payload() []byte - Ack() -} - -type Mock_MqttMessage struct { - mock.Mock -} - -func (message *Mock_MqttMessage) Duplicate() bool { - args := message.Called() - return args.Bool(0) -} - -func (message *Mock_MqttMessage) Qos() byte { - args := message.Called() - return args.Get(0).(byte) -} - -func (message *Mock_MqttMessage) Retained() bool { - args := message.Called() - return args.Bool(0) -} - -func (message *Mock_MqttMessage) Topic() string { - args := message.Called() - return args.String(0) -} - -func (message *Mock_MqttMessage) MessageID() uint16 { - args := message.Called() - return args.Get(0).(uint16) -} - -func (message *Mock_MqttMessage) Payload() []byte { - args := message.Called() - return args.Get(0).([]byte) -} - -func (message *Mock_MqttMessage) Ack() { - message.Called() -} diff --git a/test/unittest/mocks/mock_mqtt/mock_mqtttoken.go b/test/unittest/mocks/mock_mqtt/mock_mqtttoken.go deleted file mode 100644 index 683afb86..00000000 --- a/test/unittest/mocks/mock_mqtt/mock_mqtttoken.go +++ /dev/null @@ -1,31 +0,0 @@ -package mock_mqtt - -import ( - "time" - - "github.com/stretchr/testify/mock" -) - -type Mock_MqttToken struct { - mock.Mock -} - -func (token *Mock_MqttToken) Wait() bool { - args := token.Called() - return args.Bool(0) -} - -func (token *Mock_MqttToken) WaitTimeout(duration time.Duration) bool { - args := token.Called(duration) - return args.Bool(0) -} - -func (token *Mock_MqttToken) Done() <-chan struct{} { - args := token.Called() - return args.Get(0).(<-chan struct{}) -} - -func (token *Mock_MqttToken) Error() error { - args := token.Called() - return args.Error(0) -} diff --git a/test/unittest/mocks/mock_playbook_database/mock_playbook_database.go b/test/unittest/mocks/mock_playbook_database/mock_playbook_database.go index da179f40..a3dd5896 100644 --- a/test/unittest/mocks/mock_playbook_database/mock_playbook_database.go +++ b/test/unittest/mocks/mock_playbook_database/mock_playbook_database.go @@ -1,8 +1,10 @@ -package mock_playbook_database +package mock_database_controller import ( - "soarca/pkg/models/api" - "soarca/pkg/models/cacao" + "context" + "soarca/internal/store" + "soarca/internal/transport/http/schema" + "soarca/pkg/cacao" "github.com/stretchr/testify/mock" ) @@ -11,32 +13,49 @@ type MockPlaybook struct { mock.Mock } -func (testInterface *MockPlaybook) GetPlaybookMetas() ([]api.PlaybookMeta, error) { - args := testInterface.Called() - return args.Get(0).([]api.PlaybookMeta), args.Error(1) +var _ storage.PlaybookStore = (*MockPlaybook)(nil) + +// New storage interface +func (m *MockPlaybook) Create(ctx context.Context, pb cacao.Playbook) error { + args := m.Called(ctx, pb) + return args.Error(0) } -func (testInterface *MockPlaybook) GetPlaybooks() ([]cacao.Playbook, error) { - args := testInterface.Called() - return args.Get(0).([]cacao.Playbook), args.Error(1) +func (m *MockPlaybook) Update(ctx context.Context, pb cacao.Playbook) error { + args := m.Called(ctx, pb) + return args.Error(0) } -func (testInterface *MockPlaybook) Create(jsonData *[]byte) (cacao.Playbook, error) { - args := testInterface.Called(jsonData) +func (m *MockPlaybook) Get(ctx context.Context, id string) (cacao.Playbook, error) { + args := m.Called(ctx, id) return args.Get(0).(cacao.Playbook), args.Error(1) } -func (testInterface *MockPlaybook) Read(id string) (cacao.Playbook, error) { - args := testInterface.Called(id) - return args.Get(0).(cacao.Playbook), args.Error(1) +func (m *MockPlaybook) Delete(ctx context.Context, id string) error { + args := m.Called(ctx, id) + return args.Error(0) } -func (testInterface *MockPlaybook) Update(id string, jsonData *[]byte) (cacao.Playbook, error) { - args := testInterface.Called(id, jsonData) - return args.Get(0).(cacao.Playbook), args.Error(1) +func (m *MockPlaybook) List(ctx context.Context) ([]cacao.Playbook, error) { + args := m.Called(ctx) + return args.Get(0).([]cacao.Playbook), args.Error(1) } -func (testInterface *MockPlaybook) Delete(id string) error { - args := testInterface.Called(id) - return args.Error(0) +func (m *MockPlaybook) ListMeta(ctx context.Context) ([]api.PlaybookMeta, error) { + args := m.Called(ctx) + return args.Get(0).([]api.PlaybookMeta), args.Error(1) +} + +// Legacy repository methods kept for older tests that still call them. +func (m *MockPlaybook) GetPlaybookMetas() ([]api.PlaybookMeta, error) { return m.ListMeta(context.Background()) } +func (m *MockPlaybook) GetPlaybooks() ([]cacao.Playbook, error) { return m.List(context.Background()) } +func (m *MockPlaybook) Read(id string) (cacao.Playbook, error) { return m.Get(context.Background(), id) } +func (m *MockPlaybook) CreateJSON(jsonData *[]byte) (cacao.Playbook, error) { + args := m.Called(jsonData) + return args.Get(0).(cacao.Playbook), args.Error(1) +} +func (m *MockPlaybook) UpdateJSON(id string, jsonData *[]byte) (cacao.Playbook, error) { + args := m.Called(id, jsonData) + return args.Get(0).(cacao.Playbook), args.Error(1) } +func (m *MockPlaybook) DeleteLegacy(id string) error { return m.Delete(context.Background(), id) } diff --git a/test/unittest/mocks/mock_reporter/mock_downstream_reporter.go b/test/unittest/mocks/mock_reporter/mock_downstream_reporter.go index e8502458..9f1fb066 100644 --- a/test/unittest/mocks/mock_reporter/mock_downstream_reporter.go +++ b/test/unittest/mocks/mock_reporter/mock_downstream_reporter.go @@ -1,7 +1,8 @@ package mock_reporter import ( - "soarca/pkg/models/cacao" + "soarca/pkg/cacao" + "soarca/internal/runs/model" "sync" "time" @@ -14,24 +15,24 @@ type Mock_Downstream_Reporter struct { Wg *sync.WaitGroup } -func (ds_reporter *Mock_Downstream_Reporter) ReportWorkflowStart(executionId uuid.UUID, playbook cacao.Playbook, at time.Time) error { +func (ds_reporter *Mock_Downstream_Reporter) ReportWorkflowStart(runId uuid.UUID, playbook cacao.Playbook, at time.Time) error { defer ds_reporter.Wg.Done() - args := ds_reporter.Called(executionId, playbook, at) + args := ds_reporter.Called(runId, playbook, at) return args.Error(0) } -func (ds_reporter *Mock_Downstream_Reporter) ReportWorkflowEnd(executionId uuid.UUID, playbook cacao.Playbook, workflowError error, at time.Time) error { +func (ds_reporter *Mock_Downstream_Reporter) ReportWorkflowEnd(runId uuid.UUID, playbook cacao.Playbook, workflowError error, at time.Time) error { defer ds_reporter.Wg.Done() - args := ds_reporter.Called(executionId, playbook, workflowError, at) + args := ds_reporter.Called(runId, playbook, workflowError, at) return args.Error(0) } -func (ds_reporter *Mock_Downstream_Reporter) ReportStepStart(executionId uuid.UUID, step cacao.Step, stepResults cacao.Variables, at time.Time) error { +func (ds_reporter *Mock_Downstream_Reporter) ReportStepStart(metadata run.Metadata, step cacao.Step, stepResults cacao.Variables, at time.Time) error { defer ds_reporter.Wg.Done() - args := ds_reporter.Called(executionId, step, stepResults, at) + args := ds_reporter.Called(metadata, step, stepResults, at) return args.Error(0) } -func (ds_reporter *Mock_Downstream_Reporter) ReportStepEnd(executionId uuid.UUID, step cacao.Step, stepResults cacao.Variables, stepError error, at time.Time) error { +func (ds_reporter *Mock_Downstream_Reporter) ReportStepEnd(metadata run.Metadata, step cacao.Step, stepResults cacao.Variables, stepError error, at time.Time) error { defer ds_reporter.Wg.Done() - args := ds_reporter.Called(executionId, step, stepResults, stepError, at) + args := ds_reporter.Called(metadata, step, stepResults, stepError, at) return args.Error(0) } diff --git a/test/unittest/mocks/mock_reporter/mock_reporter.go b/test/unittest/mocks/mock_reporter/mock_reporter.go index dea75a36..dcf10de0 100644 --- a/test/unittest/mocks/mock_reporter/mock_reporter.go +++ b/test/unittest/mocks/mock_reporter/mock_reporter.go @@ -1,7 +1,8 @@ package mock_reporter import ( - "soarca/pkg/models/cacao" + "soarca/pkg/cacao" + "soarca/internal/runs/model" "time" "github.com/google/uuid" @@ -12,16 +13,16 @@ type Mock_Reporter struct { mock.Mock } -func (reporter *Mock_Reporter) ReportWorkflowStart(executionId uuid.UUID, playbook cacao.Playbook, at time.Time) { - _ = reporter.Called(executionId, playbook, at) +func (reporter *Mock_Reporter) ReportWorkflowStart(runId uuid.UUID, playbook cacao.Playbook, at time.Time) { + _ = reporter.Called(runId, playbook, at) } -func (reporter *Mock_Reporter) ReportWorkflowEnd(executionId uuid.UUID, playbook cacao.Playbook, err error, at time.Time) { - _ = reporter.Called(executionId, playbook, err, at) +func (reporter *Mock_Reporter) ReportWorkflowEnd(runId uuid.UUID, playbook cacao.Playbook, err error, at time.Time) { + _ = reporter.Called(runId, playbook, err, at) } -func (reporter *Mock_Reporter) ReportStepStart(executionId uuid.UUID, step cacao.Step, returnVars cacao.Variables, at time.Time) { - _ = reporter.Called(executionId, step, returnVars, at) +func (reporter *Mock_Reporter) ReportStepStart(metadata run.Metadata, step cacao.Step, returnVars cacao.Variables, at time.Time) { + _ = reporter.Called(metadata, step, returnVars, at) } -func (reporter *Mock_Reporter) ReportStepEnd(executionId uuid.UUID, step cacao.Step, returnVars cacao.Variables, err error, at time.Time) { - _ = reporter.Called(executionId, step, returnVars, err, at) +func (reporter *Mock_Reporter) ReportStepEnd(metadata run.Metadata, step cacao.Step, returnVars cacao.Variables, err error, at time.Time) { + _ = reporter.Called(metadata, step, returnVars, err, at) } diff --git a/test/unittest/mocks/mock_runstate/mock_runstate.go b/test/unittest/mocks/mock_runstate/mock_runstate.go new file mode 100644 index 00000000..eda66610 --- /dev/null +++ b/test/unittest/mocks/mock_runstate/mock_runstate.go @@ -0,0 +1,22 @@ +package mock_runstate + +import ( + runstate_model "soarca/internal/runs/state" + + "github.com/google/uuid" + "github.com/stretchr/testify/mock" +) + +type MockRunState struct { + mock.Mock +} + +func (reporter *MockRunState) GetRuns() ([]runstate_model.RunEntry, error) { + args := reporter.Called() + return args.Get(0).([]runstate_model.RunEntry), args.Error(1) +} + +func (reporter *MockRunState) GetRunReport(runKey uuid.UUID) (runstate_model.RunEntry, error) { + args := reporter.Called(runKey) + return args.Get(0).(runstate_model.RunEntry), args.Error(1) +} diff --git a/test/unittest/mocks/mock_utils/stix/mock_stix.go b/test/unittest/mocks/mock_utils/stix/mock_stix.go index c44cbcc9..8f884e83 100644 --- a/test/unittest/mocks/mock_utils/stix/mock_stix.go +++ b/test/unittest/mocks/mock_utils/stix/mock_stix.go @@ -1,7 +1,7 @@ package mock_stix import ( - "soarca/pkg/models/cacao" + "soarca/pkg/cacao" "github.com/stretchr/testify/mock" ) diff --git a/test/unittest/mocks/mock_walker/mock_walker.go b/test/unittest/mocks/mock_walker/mock_walker.go new file mode 100644 index 00000000..53eac968 --- /dev/null +++ b/test/unittest/mocks/mock_walker/mock_walker.go @@ -0,0 +1,29 @@ +package mock_walker + +import ( + "soarca/internal/workflow" + "soarca/pkg/cacao" + + "github.com/google/uuid" + "github.com/stretchr/testify/mock" +) + +type Mock_Walker struct { + mock.Mock +} + +func (mock *Mock_Walker) ExecuteAsync(playbook cacao.Playbook, results chan workflow.Result) { + args := mock.Called(playbook, results) + if results != nil { + results <- workflow.Result{ + RunId: args.Get(2).(uuid.UUID), + PlaybookId: playbook.ID, + Variables: cacao.NewVariables(), + } + } +} + +func (mock *Mock_Walker) Execute(playbook cacao.Playbook) (*workflow.Result, error) { + args := mock.Called(playbook) + return args.Get(0).(*workflow.Result), args.Error(1) +} diff --git a/tmp/refactor/durable-execution-design.md b/tmp/refactor/durable-execution-design.md new file mode 100644 index 00000000..e0ded7ff --- /dev/null +++ b/tmp/refactor/durable-execution-design.md @@ -0,0 +1,505 @@ +# Durable playbook execution — design notes + +Working document. Move it into `docs/` or `issue-drafts/` if you want it tracked. + +## Why change anything + +Four requirements the current design cannot meet: + +1. **Survive a restart.** A playbook run that is halfway through must continue after + SOARCA restarts. +2. **Resume from a step.** After a failure, re-run from the failed step — not the whole + playbook. Manual steps can take hours or days; re-running everything is not viable. +3. **Run steps in parallel.** CACAO defines a `parallel` step type. Today it is silently + skipped. +4. **Trace nested playbooks.** A sub-playbook must be traceable to the step that started it. + +## The one idea + +> **Waiting becomes a row in a database, not a goroutine blocked on a channel.** + +Everything else follows from that sentence. + +Today, "where we are in the playbook" lives in a Go call stack: a `for` loop in +`ExecuteBranch`, local variables, and recursion into sub-playbooks. A call stack cannot be +written to a database, so it cannot survive a restart. That is the whole problem. + +If instead the position is stored as data — "run 7 is waiting on step-run 12, deadline +14:05" — then a restart is uninteresting. You read the row and carry on. + +## Now vs proposed + +Today, one goroutine per run walks the graph and blocks whenever a step waits: + +```mermaid +flowchart TD + T[POST /trigger] --> D[Walker goroutine] + D --> S1[step 1: ssh] --> S2[step 2: manual] + S2 -.blocks on channel.-> W((waiting
in memory)) + W -.restart = lost.-> X[run is gone] + S2 --> S3[step 3: playbook-action] + S3 -.recursive call.-> D2[nested walker goroutine] +``` + +Proposed: a stateless loop over durable state. + +```mermaid +flowchart LR + EV[Event] --> R[reconcile run
pure function] + R --> DB[(runs
step_runs
jobs)] + R --> DIS[dispatch ready work] + DIS --> WK[stateless workers] + WK --> EV + SW[deadline sweeper] --> EV +``` + +An *event* is anything that might let a run move: it was triggered, a FIN returned a +result, an operator answered a manual step, a child run finished, or a deadline passed. + +## Data model + +Three tables. That is the whole engine's memory. + +```mermaid +erDiagram + RUN ||--o{ STEP_RUN : has + STEP_RUN ||--o{ JOB : dispatches + STEP_RUN ||--o| RUN : "spawns child (parent_step_run_id)" + + RUN { + uuid run_id + string playbook_id + string status + json variables + int version + } + STEP_RUN { + uuid step_run_id + uuid run_id + string step_id + string status + json inputs + json outputs + time deadline + uuid parent_step_run_id + } + JOB { + uuid job_id + uuid step_run_id + string capability_type + string status + time lease_expires_at + } +``` + +`RUN.parent_step_run_id` is how requirement 4 is met: a nested playbook is a normal run +that happens to know which step-run created it. Tracing is then a tree walk. + +## The loop + +One function, called whenever an event arrives: + +``` +reconcile(run): + load run + its step_runs + mark finished work, record outputs + find steps whose predecessors are all done + dispatch them (write jobs, set deadlines) + if nothing left running -> mark run finished + save with optimistic lock +``` + +It never blocks and never sleeps. It is a pure function from state to "what to do next", +which makes it trivial to unit test: feed it a state, assert the dispatch list. + +**The rule that keeps this honest: `reconcile` must never block.** The moment one step type +blocks a goroutine, durability is lost again. + +## How each step type finishes + +All four look identical to the engine — a step-run moves out of `waiting`: + +| Step type | Dispatched as | Completed by | +|---|---|---| +| ssh / http / openc2 / powershell | job for an in-process worker | worker posts result | +| fin | job on the queue | external FIN submits result | +| manual | pending command | operator `PUT /manual/{run_id}/{step_run_id}` | +| playbook-action | child run with `parent_step_run_id` | child run reaches an end state | + +This is the unification. Today these are three unrelated mechanisms (channel, queue, +recursion) with three different timeout stories (context timeout, lease + heartbeat, none +at all). Afterwards there is one mechanism and one timeout: a `deadline` column and a +sweeper that fails or retries whatever is past it. + +## Temporal vs hand-rolled + +Temporal Server and all SDKs are open source (MIT) and free to self-host. Temporal Cloud is +a paid managed option. The cost is therefore operational, not licensing: a Temporal server +plus a backing database (PostgreSQL/MySQL/Cassandra), and Elasticsearch for advanced +visibility. Local dev is a single binary (`temporal server start-dev`, SQLite). + +It maps well onto our requirements: + +| Requirement | Temporal | +|---|---| +| Survive restart | native | +| Resume from a step | `workflow reset` rewinds to a point in history | +| Parallel steps | native | +| Nested playbook traceability | child workflows, parent linkage built in | +| Manual steps lasting days | signals + durable timers — a canonical use case | + +Capabilities map onto activities almost one-to-one. FIN fits async activity completion. + +Arguments against, in order of weight: + +1. **Deployment footprint.** SOARCA ships today as one Go binary plus optional MongoDB. + Requiring adopters to run a Temporal cluster (and a second datastore, since Temporal + does not use MongoDB) is a real adoption cost for an open-source SOAR. Tracecat accepted + this trade; whether we should is a product question, and it should be answered *before* + the spike because it may decide the outcome regardless of how the code goes. +2. **Determinism and versioning discipline** — see below. +3. Another datastore to operate and back up. + +### Playbooks are data, not code — and that helps + +Temporal's model is "workflow as code", but CACAO playbooks are an interpreted graph. So we +would write *one* interpreter workflow that walks the playbook and calls activities per +step, rather than a workflow per playbook. + +This is less of a problem than it first appears, because **playbooks are uploaded and +versioned**. Pin the playbook (or its hash) as workflow input at run start: + +- Running a different playbook is *different input*, not different code. No versioning API. +- Editing a playbook after a run started cannot corrupt replay, because the run carries the + version it began with. +- Temporal's versioning tax then applies only when *we* change interpreter logic in a way + that alters the sequence of commands issued — far rarer than playbook edits. + +That reduces the second objection considerably. + +### What actually breaks determinism + +Variable interpolation is *not* a determinism problem — it is pure string replacement. The +real sources in today's code are: + +| Code | Problem | Temporal equivalent | +|---|---|---| +| `decomposer.guid.New()` (run + step ids) | random | `workflow.SideEffect` / deterministic ids | +| `decomposer.time.Now()` (all reporter calls) | wall clock | `workflow.Now()` | +| `decomposer.time.Sleep(step.Delay)` | real sleep | `workflow.Sleep` (durable timer) | +| reporter chain → TheHive HTTP | I/O inside the walk | must become an activity | +| `Variables.Merge` on key conflict | map order dependent | make merge order explicit | + +None are hard, but they must be done deliberately. The TheHive reporter is the one that +needs restructuring rather than a one-line swap. + +### Where to interpolate variables — resolve late + +Today `action.go` interpolates commands, targets **and authentication** immediately before +calling the capability (`interpolateAuthentication` covers username, password, token, +private key, oauth header). + +Interpolating early is not required, and doing it late is better for two reasons: + +1. **Secrets.** If interpolation happens in workflow code, resolved passwords, tokens and + private keys are written into Temporal's durable event history. Resolving inside the + activity keeps credentials out of persisted history. The same argument applies to our own + `step_runs.inputs` column in the hand-rolled design. +2. **Smaller determinism surface.** Less logic in workflow code means fewer ways to break + replay and fewer forced versionings. + +The tension is auditability: step reports carry `commands_b64`, and a SOAR must record what +actually ran. Resolve the command inside the activity, then return the resolved command in +the activity result for the audit trail — but redact anything that came from an +authentication field. + +Related gap: `cacao.Variable` has `Type`, `Name`, `Description`, `Value`, `Constant` and +`External`, but **no way to mark a variable sensitive**. Without that, we cannot +automatically decide what to redact from reports or history. Worth adding as a SOARCA +extension. + +### Suggested spike shape + +The `runs.Runner` boundary makes this decision reversible, so timebox hard rather than +running a two-way bake-off: + +- **Week 1, Temporal only.** One vertical slice: trigger → ssh step → manual step → kill + the server → restart → operator answers → run completes, plus a nested playbook as a + child workflow. If that lands cleanly, stop; you have the answer. +- **Only if it does not**, spend week 2 on the hand-rolled version (three tables, one + `reconcile`, a poller, a sweeper). + +Agree the decision criteria up front or the spike ends in a vibe. Proposed: does +resume-from-step work without replaying side effects; can an adopter still deploy with one +`docker compose up`; what is the upgrade burden; how hard is it to evolve the interpreter +with runs in flight; how much code do we own afterwards. + +Either option needs the explicit `in_args`/`out_args` model below. Temporal will not model +CACAO variable scoping for us. + +## Storage: SQLite locally, PostgreSQL in production + +**Decision: move off MongoDB.** In-memory/SQLite for local and test, PostgreSQL in +production, so an operator can run one database (cluster) for both SOARCA and Temporal. + +This resolves several open points at once: + +- **Removes the "second datastore" objection to Temporal.** Temporal supports PostgreSQL + and does not support MongoDB, so staying on Mongo would have forced operators to run two + database technologies. On Postgres it is one. +- **Unlocks the ergonomic durable queue.** `SELECT ... FOR UPDATE SKIP LOCKED` is the + standard pattern for exactly the claim/lease behaviour we need. On Mongo it would have + been `findAndModify` with lease fields — workable but easier to get subtly wrong. +- **Kills the `bson` tag leak.** The Phase 6 clean-up item (domain models carrying + `bson:"_id"`, notably `cacao.Playbook.ID`) disappears: playbooks are stored as JSONB and + the existing `json:` tags suffice. +- **Keeps local dev trivial.** SQLite in-process plus Temporal's single-binary dev server + means `go test ./...` and local runs need no containers at all. + +Implementation notes: + +- `internal/storage` already abstracts `Store`/`PlaybookStore`/`FinStore`, so the change is + contained to adapters. `internal/storage/mongodb` is replaced rather than modified. +- **Use a pure-Go SQLite driver (`modernc.org/sqlite`), not `mattn/go-sqlite3`.** The + makefile builds with `CGO_ENABLED=0`; a cgo driver would break the static binary and the + cross-compilation targets. +- **The hand-written in-memory store can go away.** SQLite `:memory:` gives the same + behaviour through the real SQL code path, so tests exercise the queries that production + runs instead of a parallel implementation. One adapter instead of two. +- Playbooks as JSONB give indexed queries on playbook fields, which the metadata listing + currently does by hand. + +### Schema migrations + +Mongo needed none — documents just carry whatever fields they have. SQL needs the tables to +exist, and every change (new column, new index, the `step_runs` table) must be applied to +every deployed database in order, exactly once. + +- **Tool: goose.** Lightweight, supports both SQLite and PostgreSQL, and embeds migrations + via `embed.FS`, which keeps the single-binary deployment intact. +- **Run them automatically at startup**, so `docker compose up` still needs no extra steps. +- **Dialect caveat:** SQLite's `ALTER TABLE` is limited compared to PostgreSQL. Either keep + the schema deliberately portable, or maintain dialect-specific migration sets. Decide + early; retrofitting is painful. + +### Temporal has its own schema + +"One cluster" does not mean one schema. Temporal manages its own schema with its own tool +(`temporal-sql-tool`) and wants two databases: + +``` +postgres://…/soarca <- our goose migrations +postgres://…/temporal <- Temporal's own tooling +postgres://…/temporal_visibility <- Temporal's own tooling +``` + +Same cluster, same credentials story, separate schemas. The operational win holds; the +compose file and deployment docs need to reflect it from the start. + +### No data migration from MongoDB + +Fresh installs are assumed. If an upgrade path is ever needed, signal the break with a new +major version rather than writing a Mongo→SQL converter. + + +CACAO already specifies this and we currently implement half of it. + +- `in_args` — what a step reads. **Defined in our model, never used.** +- `out_args` — what a step exports. Used today. + +Today all variables are merged into one flat map, so everything is effectively global. +That breaks two things: parallel branches writing the same variable race, and you cannot +reconstruct the inputs of step N without replaying steps 1..N-1. + +Proposed: persist per step-run the resolved `inputs` and the declared `outputs`. A step's +scope is then *run variables + outputs of its ancestors*, computed from data. + +This is the same model as GitHub Actions (`steps..outputs.`) and GitLab CI +(`artifacts:reports:dotenv` flowing to jobs that declare `needs:`). Neither lets outputs +leak implicitly. + +**This is not a nice-to-have: it is what makes requirement 2 possible.** You cannot resume +at step N unless you stored the scope entering step N. + +## Conditions, replay and the run graph + +### Reading the world is an activity + +Conditions come in two shapes: + +- **Over variables we already hold.** Today's if/while steps evaluate a STIX comparison + against `cacao.Variables` produced by earlier steps. Those values are recorded, so the + condition evaluates identically on replay. Already safe. +- **Needing fresh external state** ("is the host still infected?"). That is I/O, so it must + be an activity. It runs once, the answer is recorded, and the branch is taken on the + recorded value. + +The rule is the same one as everywhere else: **reading the world is an activity; deciding is +workflow code.** + +### Replay is not re-execution + +> Replay reconstructs what already happened. It never re-checks the world. + +If the world changed after a check, the branch does not retroactively change. That is +correct: a playbook that quarantined a host at 10:00 must not un-decide it during a replay +at 10:05. + +This is a property of **durability, not of Temporal**. In the hand-rolled design the +condition outcome is stored on the `step_run` row and resuming reads the stored value. +Identical semantics either way. + +### Stale conditions — an authoring pattern + +A run can wait days on a manual step and then evaluate a condition against data captured +before the wait. Deterministically correct, possibly operationally wrong. + +Fix in the playbook, not the engine: **when a condition must reflect current reality, put an +explicit refresh action step immediately before it.** The author controls when reality is +sampled, which is better than an engine that silently re-samples. + +To make a *running* playbook react to an external change, send it a signal (Temporal) or +raise an event that triggers `reconcile` (hand-rolled). That is also how cancellation works. + +### Stored decisions stand + +On resume, previously recorded branch decisions are authoritative. An operator who wants +fresh evaluation starts a new run. + +This matches GitLab CI and GitHub Actions: retrying a failed job re-runs that job and the +jobs that depend on it, never earlier ones. Silent divergence in a security tool's audit +log is worse than an extra button. + +### The playbook graph has cycles; the run graph does not + +A playbook may loop. A *run* never does — revisiting a step produces a **new step run**, +because causality only moves forward. + +Playbook graph (static, cyclic): + +```mermaid +flowchart LR + S[start] --> A[action a] + A --> W{while cond} + W -->|true| B[action b] + B --> W + W -->|false| E[end] +``` + +Run graph (unrolled, acyclic): + +```mermaid +flowchart LR + S1[start] --> A1["a #1"] --> W1["cond #1"] --> B1["b #1"] + B1 --> W2["cond #2"] --> B2["b #2"] --> W3["cond #3"] --> E1[end] +``` + +The current code already assumes this: `decomposer.newStepMetadata` mints a fresh +`StepExecutionId` per invocation, explicitly so that loop iterations stay distinct. + +Consequences: + +1. **Resume targets a step *run*, not a step.** If a step ran five times, "resume from step + b" is ambiguous. The API takes `step_run_id`. +2. **Loop iteration, retry and resume are one mechanism.** All three create a new step run + for the same `step_id`. A retry is just a step run with a `retry_of` link; nothing is + mutated in place, so the audit trail stays append-only. +3. **Retry invalidates descendants, not ancestors.** Step runs causally downstream of a + retried step are superseded and re-created; earlier ones are untouched. + +### Lineage + +To know what to invalidate on retry, record *which step run produced each input*. If we +store resolved `inputs` per step run (see Variables), storing the producing `step_run_id` +alongside costs almost nothing and yields a precise dependency graph. + +That is also data lineage — for a security tool, "where did this value come from?" is +worth having for its own sake. + +## Restart and resume + +On boot: find runs that are not finished, re-dispatch anything `ready`, re-arm deadlines. +Manual steps need no special handling — they were `waiting` before the restart and still +are. + +Resume-from-step after a failure: create a new step run for the failed step, leave earlier +step runs untouched, supersede those causally downstream, reconcile. Because inputs are +stored, the step can run without replaying its predecessors. + +## Concurrency + +Two events can hit one run at the same instant — for example a FIN result arriving exactly +as the sweeper decides the step timed out. Without protection both proceed and the next +step is dispatched twice. + +Fix: a `version` column. Read at version 7, compute, `UPDATE ... WHERE version = 7`. If +another writer won, zero rows change; re-read and retry. (Alternative: one consumer per +run, partitioned by `run_id`. Either works — pick one on purpose.) + +## Is this a known pattern? + +Yes. This is a **durable workflow engine**, and every part has a standard name: + +| What we are doing | Established name | Prior art | +|---|---|---| +| State in a DB, stateless loop converges it | **Reconciliation / level-triggered control loop** | Kubernetes controllers | +| Central component decides the next step | **Orchestration** (vs choreography) | Temporal, Step Functions, Conductor | +| Work claimed with a lease that expires | **Lease / visibility timeout** | SQS, our own FIN queue | +| `UPDATE ... WHERE version = n` | **Optimistic concurrency control** | standard RDBMS practice | +| Re-dispatch may repeat work | **At-least-once + idempotency keys** | Stripe, SQS | +| Rebuild state from recorded facts | **Event sourcing** | Temporal, Cadence | + +The closest mental model is a **CI system**, which is where the instinct to copy GitLab is +correct: GitLab CI is a stateless Rails app, state in Postgres, and stateless runners that +poll for jobs and hold leases. That is exactly the shape above. + +## What we are deliberately not doing + +- Not doing compensation/rollback (**Saga** pattern). Out of scope until asked for. +- Not building a distributed scheduler. One process reconciling is fine; the model allows + more later without redesign. + +## Cheapest viable version (hand-rolled option) + +This does not have to be big: + +- 3 tables +- 1 `reconcile` function +- 1 poller that claims dispatched jobs +- 1 sweeper for expired deadlines + +The FIN queue already implements claim, lease, expiry and requeue. Backing that with +storage instead of a map is most of the job. + +## Decisions + +1. **Retry is step-wise**, matching CI/CD pipelines so the behaviour is already familiar to + operators. The run-graph model supports this directly: a retry is a new step run for the + same `step_id` with a `retry_of` link, and causally downstream step runs are superseded. + Command- or target-level retry can be added later without a schema change if + `command_run` rows are recorded from the start. +2. **Idempotency is the playbook author's responsibility.** Commands have side effects + (ssh, http, manual) and the engine cannot make them repeatable. The engine's job is + narrower — see the open item below. +3. **Storage: SQLite locally, PostgreSQL in production.** A separate in-memory store is not + needed; SQLite `:memory:` covers testing through the real code path. +4. **No Mongo→SQL data migration.** Fresh installs; signal the break with a major version. + Schema migrations via goose. +5. **Stored branch decisions stand on resume.** An operator wanting fresh evaluation starts + a new run. + +## Open + +1. **The ambiguous crash window.** If SOARCA dies after dispatching a step but before + recording a result, the engine cannot know whether the side effect happened. Since + commands are not idempotent (decision 2), auto-retry is not always safe: re-running + "block this IP" is harmless, re-running "notify all users" is not. Options: mark such + step runs `unknown` and require an operator decision; or allow playbook authors to + declare a step safe to auto-retry. Recommend recording `step_run_id` as an idempotency + key on dispatch regardless, so a worker or FIN can recognise a re-delivery. +2. **Deployment footprint / Temporal.** Whether adopters must run a Temporal cluster. A + product question, and likely the deciding factor for the engine choice. Settle it before + the spike. +3. **Sensitive variables.** `cacao.Variable` has no way to mark a value secret, so we cannot + automatically decide what to redact from reports or durable history. Needs a SOARCA + extension. diff --git a/tmp/refactor/plan.md b/tmp/refactor/plan.md new file mode 100644 index 00000000..50eedbc2 --- /dev/null +++ b/tmp/refactor/plan.md @@ -0,0 +1,402 @@ +# SOARCA restructure: orchestrator / transport separation + +Working document. Lives in `./tmp` so it is not committed; move it into the repo if you +want it tracked. + +Companion: `durable-execution-design.md` covers the future execution model (durable runs, +resume, parallel steps). This document is only about the current restructure. + +## Root cause + +`*runtime.Runtime` is a service locator that is passed *into* the things it constructs. +Because the container flows downward, the dependency graph has no direction. + +``` +Runtime ──constructs──> finRegistry, playbookService, manualInbox, ... +Runtime ──passed into──> WorkflowFactory ──> f.runtime.GetInteraction(), GetCache(), ... +Runtime ──passed into──> execution.Service ──> s.runtime.GetCache().GetExecutionReport() +``` + +Every other symptom follows from this: + +- the "circular dependency" that forced `SetExecutionRuntime` (which also secretly + constructs `TriggerService`) +- `runtime.something.something` in `internal/services/execution/runtime.go` +- transport constructing the app (`httptransport.New` calls `bootstrap.New`) +- `bootstrap.Container` holding an HTTP handler (`FinHandler`) +- ceremony interfaces (`controller/database`, `controller/informer`) that exist only to + break import cycles caused by the above +- three unrelated meanings of "controller", three of "runtime" + +There is no real cycle in the domain. `Executions -> Engine -> {Interaction, Cache, +FinQueue, PlaybookStore}` is a DAG. + +**Therefore: invert dependencies first, rename and move files last.** + +## Target boundary + +Transport never sees the orchestrator. It receives a flat value struct of interfaces: + +```go +// pkg/soarca +type Operations struct { + Playbooks playbooks.Library + Executions executions.Runner + Fins fins.Registry + Work fins.Dispatch + Manual manual.Inbox +} + +func New(cfg Config) (*Orchestrator, error) +func (o *Orchestrator) Operations() Operations +func (o *Orchestrator) Close() error +``` + +```go +// cmd/soarca/main.go +cfg := config.Load() +orc := soarca.New(cfg); defer orc.Close() +srv := httpapi.New(orc.Operations(), cfg.HTTP) +srv.Run() +``` + +`Operations` has no behaviour and no infrastructure getters, so `runtime.x.y` is +structurally impossible and depth is capped at `ops.Executions.Start(ctx, ...)`. + +### What may cross the boundary + +| Concern | Owner | +|---|---| +| routes, verbs, status codes, DTOs, json/example tags, auth, CORS, TLS | transport | +| client poll-interval hints | transport | +| `context.Context` | shared (stdlib, not HTTP) | +| domain args (`cacao.Playbook`, `cacao.Variables`, `uuid.UUID`) | core | +| typed domain errors (`playbooks.ErrNotFound`, ...) | core; transport maps to codes | +| blocking until work available (`PollJob`) | core | + +## Target layout + +``` +pkg/ interfaces, domain types, errors. NO logic. + soarca/ Orchestrator, Operations, Config + cacao/ CACAO model (json + validate + example) + playbooks/ Library + errors + executions/ Runner + status/metadata types + manual/ Inbox, PendingStep, Response + fins/ Registry, Dispatch, Record + fins/protocol/ wire types, dependency-light (stdlib + uuid only) +internal/ + config/ + orchestrator/ composition root + playbooks/ (1) persistence + CRUD + executions/ (3) start/resume/status + (6) status reporting read side + engine/ decomposer construction (was bootstrap/workflow_factory.go) + fins/registry/ (2) identity, tokens, persistence [state: FinStore] + fins/dispatch/ (4) queue, leases, long-poll [state: Queue] + manual/ (5) inbox + interaction registry + adapters/storage/{memory,mongodb} + adapters/thehive/ (7) plugs into the engine reporter chain + transport/http/ server.go, routes.go, handlers/* +cmd/soarca/main.go +``` + +Everything else under today's `pkg/` (`core/`, `reporting/`, `api/`, `utils/`, +`integration/`, rest of `models/`) moves to `internal/`. + +Three merges do most of the work: + +- `trigger` + `execution` + `reporter` -> `internal/executions` (kills the fake cycle) +- `fin/registry` + `fin/work` + `capability/fin/queue` -> `internal/fins/*` +- `controller/database`, `controller/informer` -> deleted + +## Naming + +| Now | Target | +|---|---| +| `internal/runtime.Runtime` | `orchestrator.Orchestrator` | +| `services.ExecutionRuntime` | `executions.Runner` | +| `bootstrap.Container` | deleted | +| `controller.Initialize`, `decomposer_controller.IController`, `database.IController` | deleted | +| `TriggerService.ExecutePlaybook` | `executions.Runner.Start` (route stays `/trigger`) | +| `Decomposer` | `WorkflowRunner` | +| `WorkflowFactory.NewDecomposer()` | `type NewRunner func() WorkflowRunner` | +| `IDecomposer`, `ICapability`, `IActionExecutor`, `IWorkflowReporter` | drop `I` prefix | +| `InteractionController`/`ManualInbox`/`CommandInfo`/`InteractionResponse` | `manual.Inbox`, `manual.PendingStep`, `manual.Response` | +| `FinRegistry` / `FinWorkService` | `fins.Registry` / `fins.Dispatch` | +| `PlaybookService` | `playbooks.Library` | + +No documentation updates for now (per decision). + +## Phases + +Each phase compiles, keeps tests green, and is independently mergeable. + +### Phase 0 — safety net (DONE) + +See "Phase 0 results" below. + +### Phase 1 — stop passing the container (DONE) + +No files moved. `WorkflowFactory` and `execution.Service` now take explicit dependency +structs instead of `*appruntime.Runtime`: + +```go +newWorkflowFactory(EngineDeps{Interaction, Cache, FinQueue, FinStore, PlaybookStore, Config}) +execservice.New(wf, interaction, cache) +``` + +`execution.Service` declares its own narrow consumer interfaces (`ManualResumer`, +`ExecutionReports`) rather than reaching through the container. `bootstrap.New` is now +the only place that reads runtime getters, which is legitimate for a composition root. + +No service or engine holds `*Runtime` any more. It survives only in `bootstrap.New`, +`controller.go`, `httptransport.New` and tests — all removed in Phase 3. + +### Phase 2 — collapse the fake cycle (DONE) + +Merged `trigger` + `execution` + `reporter` into `internal/executions`, exposing one +interface: + +```go +type Runner interface { + Start(ctx, playbook, variables) (uuid.UUID, error) + StartByID(ctx, playbookID, variables) (uuid.UUID, error) + List(ctx) ([]cache.ExecutionEntry, error) + Report(ctx, executionID) (cache.ExecutionEntry, error) +} +``` + +The engine moved to `internal/executions/engine` — required, not cosmetic: `bootstrap` +imports `runtime`, so `runtime` could not import the factory while it lived in +`bootstrap`. With the engine outside, `Runtime` now constructs strictly top-down: + +``` +storage -> cache/interaction/queue -> engine.New(deps) -> executions.New(engine, store, cache) +``` + +Deleted: `SetExecutionRuntime`, `runtime.triggerService = nil`, `GetExecutionRuntime`, +`GetTriggerService`, `GetReporterService`, `internal/services/{execution,trigger,reporter}`, +`internal/controller/informer`, `internal/bootstrap/workflow_factory.go`, and the +`ExecutionRuntime` / `TriggerService` / `ReporterService` interfaces. + +`bootstrap.New` no longer constructs anything; it only builds `TransportOptions`. +Phase 3 deletes it entirely. + +Dead code removed in the process (no production callers, found by usage audit): + +- `ExecutionRuntime.ResumeManualStep` — duplicated `ManualInbox.ContinuePendingCommand`; + both ended at `interaction.PostContinue`. The manual handler only ever used the inbox. +- `ExecutionRuntime.GetExecutionStatus` — duplicated `ReporterService.GetExecutionReport`. + +Also fixed here (were listed as known defects): + +- the discarded `variables` argument — `Start` now applies the variables it is given + instead of `_ = variables` +- the dead channel-filter loop over a buffered size-1 channel written by exactly one + decomposer — replaced with a plain select + +Note: `runtime.Options` gained `HTTP` and `TheHive`, and `controller.Initialize` now +passes them. Without that, `SkipCertValidation` and the whole TheHive integration would +have silently stopped being configured. + +### Phase 3 — install the boundary (DONE) + +`Runtime` now exposes exactly one surface: + +```go +type Operations struct { + Playbooks services.PlaybookService + Executions executions.Runner + Fins services.FinRegistry + Work services.FinWorkService + Manual services.ManualInbox +} + +func (r *Runtime) Operations() Operations +``` + +All infrastructure fields and getters went private (`playbookStore`, `finStore`, +`finQueue`, `cache`, `interaction`). `GetPlaybookStore`, `GetCache`, `GetFinQueue`, +`GetInteraction`, `GetFinStore` are gone, so transport cannot reach through the +container even by accident. + +`httptransport.New(ops, opts)` now takes `Operations` plus its own narrow `Options` +(Server, Fin, Auth, CORS) instead of `*Runtime` + the whole `config.Config`. Note what +is no longer passed to transport: storage config, TheHive config and outbound TLS +settings — all orchestrator concerns. The transport also constructs its own FinHandler. + +`internal/bootstrap` is deleted entirely (Container, TransportOptions, workflow factory). + +`internal/controller/controller.go` was kept rather than folded into `main.go`: it is the +composition root and its `loadConfig`/`newRuntime`/`newTransport` seams are what make +`controller_test.go` possible. It should be renamed (`internal/app`) in Phase 4 rather +than deleted — the objection was three meanings of "controller", not this file's job. + +`boundary_test.go` was rewritten: it asserted every getter returned non-nil, which pinned +the service-locator shape. It now asserts `Operations` is fully populated and documents +the getters that must not come back. + +### Phase 4 — renames and moves only (IN PROGRESS) + +#### 4a — test layout (DONE) + +Idiomatic Go puts `foo_test.go` beside `foo.go` in the same directory; a top-level +`test/` tree is not a Go convention. 43 of 58 test files were already beside their code. + +Tests needing external services are now selected by build tag rather than by directory: + +- `//go:build integration` — needs `deployments/docker/testing` (httpbin, ssh) +- `//go:build manual` — needs a special environment (Windows/PowerShell host, live TheHive) + +`go test ./...` is now green for the first time. Previously six suites failed by design +on any machine without those services, which trains everyone to ignore a red suite. +Makefile targets: `test` (default, no services), `integration-test`, `manual-test`. + +Still to do in 4a: move the remaining `test/integration/api/**` suites next to the code +they exercise, turn `test/unittest/mocks` into per-package mocks, and move playbook JSON +fixtures into `testdata/` (which the go tool ignores by convention). + +#### 4b — package renames (MOSTLY DONE — remainder folded into the new engine work) + +Decision: option 2 — `run` everywhere including the wire. The FIN protocol is alpha, so +breaking changes are acceptable and no compatibility shim is needed. + +Done: + +- `internal/executions` → `internal/runs`; `executions.Runner` → `runs.Runner`. +- Wire vocabulary renamed: `execution_id` → `run_id`, `step_execution_id` → `step_run_id`, + `"execution_status"` → `"run_status"`. Route params `/manual/:exec_id/:step_execution_id` + → `/manual/:run_id/:step_run_id`. +- API models: `PlaybookExecutionReport` → `PlaybookRunReport`, `StepExecutionReport` → + `StepRunReport`, `api.Execution` → `api.RunStarted`. +- `fin.Job.ExecutionId`/`StepExecutionId` → `RunId`/`StepRunId` (FIN protocol break). +- Fixed the `json:"payload"` bug on the trigger response; now `playbook_id`. +- Swagger regenerated: 0 occurrences of `execution_id`. +- `pkg/core/decomposer` → `internal/workflow`; `Decomposer`/`IDecomposer` → + `workflow.Walk`/`workflow.Walker`. +- `decomposer_controller.IController` → `workflow.NewWalker` func type (package deleted). +- `internal/controller` → `internal/app`; `Initialize()` → `Run()`. +- `pkg/api/middelware` → `pkg/api/middleware` (typo). +- Deleted resurrected dead packages `internal/bootstrap` and `internal/services/execution`. + +Deliberately NOT done — this code is replaced by the durable engine, so renaming it is +throwaway work: + +- ~360 references to the `execution` noun (`execution.Metadata`, `ExecutionId`, + `cache.ExecutionEntry`). Concentrated in the walker, executors and the cache — all of + which the run/step-run tables replace. +- 28 `I`-prefixed interfaces, mostly in `pkg/core/executors` and the reporter chain. +- `pkg/core/executors/playbook_action` and `pkg/reporting/reporter/downstream_reporter` + underscore package names. +- `pkg/` → `internal/` split and the `pkg/soarca` embeddable entrypoint. Premature until + the engine settles. + +Residual test-layout work (moving `test/integration/api` beside the code, per-package +mocks, `testdata/`) is cosmetic; those suites are the safety net for the engine migration +and are more useful left working than moved. + +## Status + +Phases 0, 1, 2, 3 and 5 are complete. Phase 4 is complete for everything on the stable +surface. **The refactor has met its goal**: the orchestrator is separated from transport, +the boundary is enforced by `test/architecture/boundary_test.go`, and `runs.Runner` is the +seam the new engine plugs into. + +Next work is the durable execution engine — see `durable-execution-design.md`. Remaining +naming debt lives in code that work replaces, so it should be picked up there rather than +polished first. + +### Phase 5 — enforce (DONE, ahead of Phase 4) + +`test/architecture/boundary_test.go` runs `go list -deps` over the orchestrator packages +and fails if any of them transitively depends on gin, gauth, swagger, +`soarca/internal/transport` or `soarca/pkg/api`. Currently passing: the core is genuinely +transport-free. + +`net/http` is deliberately *not* forbidden — the http and openc2 capabilities make +outbound calls and legitimately need it. The rule targets inbound web framework, routing +and auth middleware. + +A second test (`TestDetectorWorks`) asserts that the transport layer *does* depend on +gin, so the check cannot silently degrade into one that inspects nothing. + +### Phase 6 — optional, non-blocking + +Strip `bson:` tags from domain models; the mongo adapter owns document types and +mapping. Notably `cacao.Playbook.ID` is `bson:"_id" json:"id"` today. + +## Known defects found while planning + +- [x] `config.Load()` panicked: `v.SetEnvKeyReplacer(nil)` overwrote viper's default + replacer, so `getEnv` nil-dereferenced. Viper's default is already a no-op. + **Production bug**, fixed in Phase 0. +- [x] `execution.Service.StartExecution` did `_ = variables`, silently discarding the + argument. Fixed in Phase 2. +- [x] Same function looped on `details.PlaybookId != playbook.ID` over a buffered + size-1 channel written by exactly one decomposer. Dead logic, removed in Phase 2. +- [ ] `pkg/models/api.Execution.PlaybookId` is tagged `json:"payload"` — wrong wire name. +- [ ] `fin.Record` is simultaneously persistence type (`bson:"_id"`), admin wire type + (`ListFins` returns it straight to a handler) and secret holder (`FinTokenHash`), + kept off the wire only by a `json:"-"` tag. Split into public `fins.Record` and an + internal persistence record. + +## Phase 0 results + +Goal: a safety net that tests the real wiring. It was red and it was pinning the old +architecture. + +Fixed: + +1. `internal/config/config.go` — removed `v.SetEnvKeyReplacer(nil)`. This panicked + `config.Load()`, which took down the whole-app smoke test in + `test/integration/api`. +2. `reporter_api_invocation_test.go` — fixture predated the per-invocation + `StepExecutionId`. It also keyed `StepResults` by `StepId` despite + `pkg/models/cache/cache.go` documenting the key as `StepExecutionId`. Both corrected. +3. `manual_api_test.go` — expected `null` for `commands`/`targets` where the handler + emits `[]`. +4. Migrated `playbook_api_test.go` (6 sites) and `reporter_api_test.go` / + `reporter_api_invocation_test.go` (4 sites) off the legacy controller-based route + helpers onto `PlaybookRoutesWithService` / `ReporterRoutesWithService`. The tests + were the *only* remaining users of the legacy path, which is why + `controller/database` still existed. +5. Deleted from `pkg/api/api.go`: `Database()`, `Reporter()`, `Api()`, `PlaybookRoutes()`, + `ReporterRoutes()`. Deleted `internal/controller/database/` and + `test/unittest/mocks/mock_controller/database/`. + +Status: `go vet ./...` clean. All packages pass except the pre-existing +external-dependency suites, which need containers and failed identically before these +changes: + +``` +pkg/utils/http (httpbin) +test/integration/capability/http (httpbin) +test/integration/capability/ssh (ssh server) +test/manual/powershell (windows host) +test/manual/thehive_connector (thehive) +test/manual/thehive_reporter (thehive) +``` + +Coverage gaps to be aware of during the refactor: `/status` has no integration coverage. + +## FIN route coverage (added) + +`test/integration/api/routes/fin_api/fin_api_test.go` — 16 tests over the real registry +and work service backed by in-memory storage and a real queue, so the route/middleware/ +service composition is covered before Phase 3 moves FIN handler construction into +transport. Covers registration (success, wrong token, no capabilities, registration +disabled), bearer auth (missing, unknown), poll with no work, result submission errors, +admin list/get/delete, and unregister. + +Includes `TestListFinsDoesNotLeakTokenHash`, a regression guard for `fin.Record` doubling +as persistence type and admin wire type with only `json:"-"` keeping the credential +hash off the wire. + +## Notes for later phases + +- `internal/controller/boundary_test.go` asserted that every `Runtime` getter returns + non-nil. It pinned the service-locator shape. Rewritten in Phase 3. +- Something in the editor/toolchain repeatedly prepends a duplicate `package X` line to + newly created Go files, producing `expected declaration, found 'package'`. Hit on + `fin_api_test.go`, `engine.go`, `service.go` and `boundary_test.go`. Check the first + two lines of any new file before building.