diff --git a/.gitignore b/.gitignore index 38ca540..f8e2863 100755 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ setup/master/.env # Data and storage directories setup/splunk/data/ +setup/elastic/data/ postgres_data/ share/smb_test_file share/cases/* @@ -51,3 +52,6 @@ share/log/task_traces.jsonl test.py share/log/celery.log OSIR/src/osir_web/osir_web/pages/dynamic/index.html + +# IA agent +.opencode \ No newline at end of file diff --git a/OSIR/bin/json2elastic-rs b/OSIR/bin/json2elastic-rs new file mode 100755 index 0000000..d25f49f Binary files /dev/null and b/OSIR/bin/json2elastic-rs differ diff --git a/OSIR/configs/modules/elastic/indexer_elastic.yml b/OSIR/configs/modules/elastic/indexer_elastic.yml new file mode 100644 index 0000000..d564eee --- /dev/null +++ b/OSIR/configs/modules/elastic/indexer_elastic.yml @@ -0,0 +1,34 @@ +metadata: + version: "1.0" + author: Grand-Duc + description: ElasticSearch ingestion (DFIR ORC and UAC) using the same module-specific + datamodel and VRL normalization as the Splunk indexer. + os: generic + +configuration: + module: indexer_elastic + type: post_parsing + disk_only: false + no_multithread: false + processor_type: + - internal + processor_os: unix + +tool: + path: json2elastic-rs + cmd: >- + --input {input_dir_replaced_by_internal_module} + --index {case_name} + --config /OSIR/setup/conf/agent.yml + --indexer_patterns {indexer_path} + --vrl_dir /OSIR/OSIR/configs/dependencies/ + source: https://github.com/Grand-Duc/json2elastic-rs/releases + version: 1.0 + +input: + type: dir + path: "{case_path}" + +output: + type: single_file + format: jsonl diff --git a/OSIR/src/osir_lib/osir_lib/core/OsirAgentConfig.py b/OSIR/src/osir_lib/osir_lib/core/OsirAgentConfig.py index 6ab539a..2d712c8 100644 --- a/OSIR/src/osir_lib/osir_lib/core/OsirAgentConfig.py +++ b/OSIR/src/osir_lib/osir_lib/core/OsirAgentConfig.py @@ -80,16 +80,38 @@ class SplunkConfig(BaseModel): ssl: bool # booléen pour True/False +class ElasticConfig(BaseModel): + """ + Contains the authentication and connection parameters for ElasticSearch integration. + + Args: + host (str): The address of the ElasticSearch server (default 127.0.0.1). + user (str): Username for ElasticSearch (default elastic). + password (str): Password for ElasticSearch. + port (int): The destination port for the REST API (default 9200). + ssl (bool): Whether to use encrypted HTTPS for the connection. + kibana_port (int): The Kibana web interface port (default 5601). Display-only, + used to build the "Open Kibana" link in the web sidebar. + """ + host: str + user: str + password: str + port: int + ssl: bool + kibana_port: int = 5601 + + class FullAgentConfig(BaseModel): """ The root validation model for the 'agent.yml' configuration file. It acts as a single point of truth, validating that the Master, Windows Box, - and Splunk sections are correctly formatted and present before the agent starts. + Splunk and ElasticSearch sections are correctly formatted and present before the agent starts. """ master: MasterConfig windows_box: WindowsBoxConfig splunk: SplunkConfig + elasticsearch: ElasticConfig @singleton @@ -204,6 +226,30 @@ def splunk_mport(self) -> int: def splunk_ssl(self) -> bool: return self.config_data.splunk.ssl + @property + def elastic_host(self) -> str: + return self.config_data.elasticsearch.host + + @property + def elastic_user(self) -> str: + return self.config_data.elasticsearch.user + + @property + def elastic_password(self) -> str: + return self.config_data.elasticsearch.password + + @property + def elastic_port(self) -> int: + return self.config_data.elasticsearch.port + + @property + def elastic_ssl(self) -> bool: + return self.config_data.elasticsearch.ssl + + @property + def elastic_kibana_port(self) -> int: + return self.config_data.elasticsearch.kibana_port + def _is_standalone(self) -> bool: """ Determines if the Agent is running on the same physical or virtual host as the Master. diff --git a/OSIR/src/osir_lib/osir_lib/modules/indexer/indexer_elastic.py b/OSIR/src/osir_lib/osir_lib/modules/indexer/indexer_elastic.py new file mode 100644 index 0000000..9e3f497 --- /dev/null +++ b/OSIR/src/osir_lib/osir_lib/modules/indexer/indexer_elastic.py @@ -0,0 +1,83 @@ +import os +from osir_lib.core.OsirDecorator import osir_internal_module +from osir_lib.core.model.OsirModuleModel import OsirModuleModel +from osir_lib.core.OsirModule import OsirModule +from osir_lib.logger import AppLogger, CustomLogger + +logger: CustomLogger = AppLogger().get_logger() + + +@osir_internal_module +class InjectionModule(): + """ + PyModule to inject parsed logs into ElasticSearch. + + Mirrors the Splunk ``indexer_ng`` internal module: it iterates every child + module output directory and, for each child that carries an indexing + datamodel (``splunk:`` block, reused as-is for ElasticSearch since it is + SIEM-agnostic), runs the ``json2elastic-rs`` tool via the framework tool + runner. + """ + + def __init__(self, module: OsirModule): + """ + Initializes the Module. + + Args: + module (OsirModule): Instance of OsirModule containing configuration details for the extraction process. + """ + self.module = module + self.case_path = module.input.match + + def __call__(self) -> bool: + """ + Execute the internal processor of the module. + + Returns: + bool: True if the processing completes successfully, False otherwise. + """ + try: + logger.debug(f"Processing case {self.case_path}") + for module_dir_name in os.listdir(self.case_path): + if os.path.isdir(os.path.join(self.case_path, module_dir_name)): + try: + child_module_model = OsirModuleModel.from_name(module_dir_name) + except FileNotFoundError: + logger.warning(f"Skipping directory '{module_dir_name}' — no matching module found.") + continue # Skip to next directory + if child_module_model: + logger.debug(f"Module found {module_dir_name}") + + # Dump model to add mandatory fields before validation : case_path and match + child_module_data = child_module_model.model_dump() + child_module_data["case_path"] = self.module.case_path + child_module_data["input"]["match"] = os.path.join(self.case_path, module_dir_name) + + # Transform model to instance. Mandatory to get _module_filepath + child_module_instance = OsirModule.model_validate(child_module_data) + + replacements = { + "indexer_path": child_module_instance._module_filepath, + "input_dir_replaced_by_internal_module": os.path.join(self.case_path, module_dir_name) + } + tool_with_place_holders = self.module.tool.cmd + + # Backup the original command for next modules to ingest + self.module.tool.cmd = self.module.tool.safe_format(self.module.tool.cmd, **replacements) + + # Run json2elastic-rs if an indexing datamodel exists. The + # ``splunk:`` block is reused as-is since it is SIEM-agnostic + # (file-matching patterns, timestamp extraction, VRL scripts); + # a dedicated ``elastic:`` block is also supported if present. + indexer_data = child_module_instance.splunk or getattr(child_module_instance, "elastic", None) + if indexer_data: + self.module.tool.run() + # Restore cmd with place holders + self.module.tool.cmd = tool_with_place_holders + else: + logger.warning(f"Module not found {module_dir_name}") + + logger.debug(f"{self.module.module_name} done") + + except Exception as exc: + logger.error_handler(exc) diff --git a/OSIR/src/osir_web/osir_web/utils/OsirWebSidebar.py b/OSIR/src/osir_web/osir_web/utils/OsirWebSidebar.py index 6250d85..1c7794d 100755 --- a/OSIR/src/osir_web/osir_web/utils/OsirWebSidebar.py +++ b/OSIR/src/osir_web/osir_web/utils/OsirWebSidebar.py @@ -63,20 +63,26 @@ def sidebar(): # time.sleep(1) # host = location_details["hostname"] - # Get splunk host + # Get splunk and elastic hosts try: agent_config = OsirAgentConfig() host = agent_config.master_host splunk_host = agent_config.splunk_host if agent_config.splunk_host not in ["host.docker.internal", "127.0.0.1"] else host + elastic_host = agent_config.elastic_host if agent_config.elastic_host not in ["host.docker.internal", "127.0.0.1"] else host + splunk_port = agent_config.splunk_port + kibana_port = agent_config.elastic_kibana_port except FileNotFoundError: # agent.yml missing -> happens if master launched before agent is installed host = "localhost" splunk_host = host - + elastic_host = host + splunk_port = 8000 + kibana_port = 5601 - # Set the URL for the iframe + # Set the URLs url = f"http://{host}:80/" - splunk_url = f"http://{splunk_host}:8000/" + splunk_url = f"http://{splunk_host}:{splunk_port}/" + kibana_url = f"http://{elastic_host}:{kibana_port}/" colored_header( label="Useful links", description="", @@ -85,6 +91,7 @@ def sidebar(): with st.expander(":round_pushpin: External"): st.page_link(url, label="Open Database", help="open a new tab to pgadmin", width='stretch', icon="💾") st.page_link(splunk_url, label="Splunk", help="open a new tab to local Splunk server", width='stretch', icon="💹") + st.page_link(kibana_url, label="Kibana", help="open a new tab to local Kibana server", width='stretch', icon="📊") # Master specs colored_header( diff --git a/README.md b/README.md index 2f339a0..983af93 100755 --- a/README.md +++ b/README.md @@ -9,15 +9,20 @@ OSIR is a dockerized and distributed parsing framework that works on Linux and W # Table of Contents +- [OSIR](#osir) +- [Table of Contents](#table-of-contents) - [Architecture](#architecture) - [How does it work ?](#how-does-it-work-) -- [Quick Start](#quick-start) -- [Contributing](#contributing) +- [Quick start](#quick-start) + - [Clone the project including dependencies](#clone-the-project-including-dependencies) + - [Example of usage: parsing DFIR ORC triage on Ubuntu host](#example-of-usage-parsing-dfir-orc-triage-on-ubuntu-host) - [Main features](#main-features) - [Documentation](#documentation) -- [Currently supported modules](#currently-supported-modules) +- [Supported Modules](#supported-modules) +- [Contributing](#contributing) - [Creators](#creators) - [License](#license) +- [Other references](#other-references) # Architecture @@ -73,6 +78,7 @@ git clone --recurse-submodules https://github.com/maxspl/OSIR - Dockerized installation - Modular: processing tasks are defined by easily modifiable configuration files - Splunk integration for output analysis +- ElasticSearch (ELK) integration for output analysis (choose Splunk, ElasticSearch or both at master setup) # Documentation @@ -84,7 +90,8 @@ Project documentation: https://osir.readthedocs.io | OS | Filename | Description | Author | Version | Processor Type | Tool Path | |:--------|:-------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------|:---------------|----------:|:-------------------|:------------------------------------| | generic | age_decrypt.yml | Used to decrypt age files. Don't forget to put the key in /OSIR/OSIR/configs/dependencies/encryption/key.age. | maxspl | 1 | external | age | -| generic | indexer-ng.yml | Splunk logs ingestion (DFIR ORC and UAC) using module-specific json2splunk-rs configuration. | maxspl | 1 | internal | json2splunk-rs | +| generic | indexer-ng.yml | Splunk logs ingestion (DFIR ORC and UAC) using module-specific json2splunk-rs configuration. | maxspl | 1 | internal | json2splunk-rs | +| generic | indexer_elastic.yml | ElasticSearch logs ingestion (DFIR ORC and UAC) using module-specific json2elastic-rs configuration. | Grand-Duc | 1 | internal | json2elastic-rs | | generic | mongodb.yml | Splunk logs ingestion of Mongodb logs. | Typ | 1 | external | json2splunk-rs | | generic | thor_lite.yml | Scan of collected file using Thor Lite. | typ | 1 | external | thor-lite/thor-lite-linux-64 | | generic | thor_orc.yml | Scan of collected DFIR ORC (output of restore_fs module) file using Thor (requires Forensic license). | maxspl | 1 | external | thor/thor-linux-64 | diff --git a/requirements.txt b/requirements.txt index 91c9ff1..bd6f9b3 100755 --- a/requirements.txt +++ b/requirements.txt @@ -24,3 +24,4 @@ python-dotenv rich requests tabulate +elasticsearch diff --git a/setup/agent/agent_setup.sh b/setup/agent/agent_setup.sh index f9daaf0..99978fe 100755 --- a/setup/agent/agent_setup.sh +++ b/setup/agent/agent_setup.sh @@ -440,6 +440,14 @@ install_from_conf(){ # Get Splunk ssl splunk_ssl=$(get_yml_value "" splunk_ssl) + # Get ElasticSearch connection details + es_host=$(get_yml_value "" es_host) + es_user=$(get_yml_value "" es_user) + es_password=$(get_yml_value "" es_password) + es_port=$(get_yml_value "" es_port) + es_ssl=$(get_yml_value "" es_ssl) + kibana_port=$(get_yml_value "" kibana_port) + (echo >&2 "${INFO} Splunk host : $splunk_host") (echo >&2 "${INFO} Splunk user : $splunk_user") (echo >&2 "${INFO} Splunk password : $splunk_password") @@ -476,6 +484,22 @@ install_from_conf(){ (echo >&2 "${ERROR} Wrong location. Needs to be local or remote.") exit 0 fi + + # Export Splunk vars with fallback defaults (even if splunk not selected, agent.yaml needs them) + export SPLUNK_REMOTE_HOST=${splunk_host:-host.docker.internal} + export SPLUNK_USER=${splunk_user:-admin} + export SPLUNK_PASSWORD=${splunk_password:-DFIR_passwd} + export SPLUNK_PORT=${splunk_port:-8000} + export SPLUNK_MPORT=${splunk_mport:-8089} + export SPLUNK_SSL=${splunk_ssl:-False} + + # Export ElasticSearch vars read from config so they propagate to agent.yml + export ELASTIC_HOST=${es_host:-host.docker.internal} + export ELASTIC_USER=${es_user:-elastic} + export ELASTIC_PASSWORD=${es_password:-DFIR_passwd} + export ELASTIC_PORT=${es_port:-9200} + export ELASTIC_SSL=${es_ssl:-False} + export KIBANA_PORT=${kibana_port:-5601} } manual_install(){ @@ -632,18 +656,26 @@ manual_install(){ mount_point="" fi - # Ask user : connect agent to Splunk server - default_connect_splunk="yes" - read -p "$(echo -n >&2 "${USERINPUT} Do you want to connect agent to a Splunk server ? [Default is: $default_connect_splunk] [options: yes/no]: ")" connect_splunk - # Initialize Splunk vars even if user answers "no" + # Ask user : which SIEM to use for visualization (mirrors master setup) + default_siem="splunk" + read -p "$(echo -n >&2 "${USERINPUT} Which SIEM do you want the agent to forward data to ? [Default is: $default_siem] [options: splunk/elasticsearch/both]: ")" siem_selected + if [[ -z "$siem_selected" ]]; then + siem_selected="$default_siem" + fi + siem_selected=$(echo "$siem_selected" | tr '[:upper:]' '[:lower:]') + if [[ "$siem_selected" != "splunk" && "$siem_selected" != "elasticsearch" && "$siem_selected" != "both" ]]; then + (echo >&2 "${ERROR} Invalid SIEM choice. Use splunk, elasticsearch or both.") + exit 0 + fi + + # Initialize Splunk vars splunk_host="" splunk_user="" splunk_password="" splunk_port="" splunk_mport="" splunk_ssl="" - if [ -z "$connect_splunk" ] || [ "$connect_splunk" = "yes" ] ; then - + if [ "$siem_selected" = "splunk" ] || [ "$siem_selected" = "both" ] ; then # Ask user for the Splunk host default_splunk_host="host.docker.internal" echo "Enter the Splunk Host:" @@ -663,7 +695,7 @@ manual_install(){ if [[ -z "$splunk_user" ]]; then splunk_user="$default_user" fi - + # Ask user : remote Splunk password default_password="DFIR_passwd" read -p "$(echo -n >&2 "${USERINPUT} Enter the Splunk password. [Default is: $default_password]: ")" splunk_password @@ -693,18 +725,81 @@ manual_install(){ fi fi + # Initialize ElasticSearch vars + es_host="" + es_user="" + es_password="" + es_port="" + es_ssl="" + kibana_port="" + if [ "$siem_selected" = "elasticsearch" ] || [ "$siem_selected" = "both" ] ; then + # Ask user for the ElasticSearch host + default_es_host="host.docker.internal" + echo "Enter the ElasticSearch Host:" + echo " - Press ENTER to use the default ElasticSearch service ($default_es_host)" + echo " - If using a remote master, enter the FQDN/IP of the remote master" + echo " - If you prefer to use a different ElasticSearch server, enter its FQDN/IP." + read -p "ElasticSearch host [Default: $default_es_host]: " es_host + es_host=${es_host:-$default_es_host} + + if [[ -z "$es_host" ]]; then + es_host="$default_es_host" + fi + + # Ask user : ElasticSearch user + default_es_user="elastic" + read -p "$(echo -n >&2 "${USERINPUT} Enter the ElasticSearch user. [Default is: $default_es_user]: ")" es_user + if [[ -z "$es_user" ]]; then + es_user="$default_es_user" + fi + + # Ask user : ElasticSearch password + default_es_password="DFIR_passwd" + read -p "$(echo -n >&2 "${USERINPUT} Enter the ElasticSearch password. [Default is: $default_es_password]: ")" es_password + if [[ -z "$es_password" ]]; then + es_password="$default_es_password" + fi + + # Ask user : ElasticSearch port + default_es_port="9200" + read -p "$(echo -n >&2 "${USERINPUT} Enter the ElasticSearch REST port. [Default is: $default_es_port]: ")" es_port + if [[ -z "$es_port" ]]; then + es_port="$default_es_port" + fi + + # Ask user : ElasticSearch SSL + default_es_ssl="False" + read -p "$(echo -n >&2 "${USERINPUT} Enable SSL for ElasticSearch communication ? . [Default is: $default_es_ssl] [options: True/False]: ")" es_ssl + if [[ -z "$es_ssl" ]]; then + es_ssl="$default_es_ssl" + fi + + # Ask user : Kibana web port (display only, used for the web sidebar link) + default_kibana_port="5601" + read -p "$(echo -n >&2 "${USERINPUT} Enter the Kibana web port. [Default is: $default_kibana_port]: ")" kibana_port + if [[ -z "$kibana_port" ]]; then + kibana_port="$default_kibana_port" + fi + fi + # Export users input to env export LOCATION_TYPE=$location_type export WINDOWS_HOST=$host export WINDOWS_USER=$user export WINDOWS_PASSWORD=$password export WINDOWS_MOUNT_POINT=${mount_point//:} # Save drive letter without ":" - export SPLUNK_REMOTE_HOST=$splunk_host - export SPLUNK_USER=$splunk_user - export SPLUNK_PASSWORD=$splunk_password - export SPLUNK_PORT=$splunk_port - export SPLUNK_MPORT=$splunk_mport - export SPLUNK_SSL=$splunk_ssl + export SPLUNK_REMOTE_HOST=${splunk_host:-host.docker.internal} + export SPLUNK_USER=${splunk_user:-admin} + export SPLUNK_PASSWORD=${splunk_password:-DFIR_passwd} + export SPLUNK_PORT=${splunk_port:-8000} + export SPLUNK_MPORT=${splunk_mport:-8089} + export SPLUNK_SSL=${splunk_ssl:-False} + export ELASTIC_HOST=${es_host:-host.docker.internal} + export ELASTIC_USER=${es_user:-elastic} + export ELASTIC_PASSWORD=${es_password:-DFIR_passwd} + export ELASTIC_PORT=${es_port:-9200} + export ELASTIC_SSL=${es_ssl:-False} + export KIBANA_PORT=${kibana_port:-5601} # Save setup config conf_agent_sample="$CONF_PATH/agent_sample.yml" diff --git a/setup/agent/dockerfile/AgentFile b/setup/agent/dockerfile/AgentFile index 6f786db..df2754f 100755 --- a/setup/agent/dockerfile/AgentFile +++ b/setup/agent/dockerfile/AgentFile @@ -4,7 +4,7 @@ FROM ubuntu:24.04 # - Environment -------------------------------- ENV DEBIAN_FRONTEND=noninteractive \ TZ=Europe/Minsk \ - PATH="/OSIR/OSIR:${PATH}" + PATH="/OSIR/OSIR:/OSIR/OSIR/bin:${PATH}" # - Base system & tooling --------------------------- RUN set -eux; \ @@ -58,7 +58,8 @@ RUN set -eux; \ # - Python dependencies (cached layer) --------------------- COPY sources/requirements.txt /tmp/requirements.txt RUN set -eux; \ - python -m pip install --no-cache-dir --break-system-packages --ignore-installed -r /tmp/requirements.txt + python -m pip install --no-cache-dir --break-system-packages --ignore-installed -r /tmp/requirements.txt; \ + python3.12 -m pip install --no-cache-dir --break-system-packages elasticsearch # - Install DECODE from https://github.com/ANSSI-FR/DECODE ----------------------- RUN set -eux; \ diff --git a/setup/agent/dockerfile/sources/requirements.txt b/setup/agent/dockerfile/sources/requirements.txt index 8e7b9cf..dfda7fc 100755 --- a/setup/agent/dockerfile/sources/requirements.txt +++ b/setup/agent/dockerfile/sources/requirements.txt @@ -27,4 +27,5 @@ dissect xxhash zat uv -sqlalchemy \ No newline at end of file +sqlalchemy +elasticsearch \ No newline at end of file diff --git a/setup/conf/agent_sample.yml b/setup/conf/agent_sample.yml index 6e5ee6e..2d055f1 100755 --- a/setup/conf/agent_sample.yml +++ b/setup/conf/agent_sample.yml @@ -7,6 +7,13 @@ splunk: port: {splunk_port} mport: {splunk_mport} ssl: {splunk_ssl} +elasticsearch: + host: {es_host} + user: {es_user} + password: {es_password} + port: {es_port} + ssl: {es_ssl} + kibana_port: {kibana_port} # Kibana web interface port (display only, used by the web sidebar link) windows_box: location: {windows_box_location} cores: {windows_box_cores} diff --git a/setup/conf/master_sample.yml b/setup/conf/master_sample.yml index f0b8294..227eee2 100755 --- a/setup/conf/master_sample.yml +++ b/setup/conf/master_sample.yml @@ -1,3 +1,5 @@ +siem: + selected: {siem_selected} # splunk | elasticsearch | both splunk: location: {splunk_location} user: {splunk_user} @@ -6,6 +8,18 @@ splunk: mport: {splunk_mport} ssl: {splunk_ssl} local_splunk: - previous_data: keep # values: erase, keep, stop + previous_data: {local_splunk_previous_data} # values: erase, keep, stop remote_splunk: host: {splunk_remote_splunk_host} +elasticsearch: + location: {es_location} # local | remote + host: {es_host} + port: {es_port} + user: {es_user} + password: {es_password} + ssl: {es_ssl} + kibana_port: {kibana_port} # Kibana web interface port (display only, used by the web sidebar link) + local_es: + previous_data: {local_elastic_previous_data} # values: erase, keep, stop + remote_es: + host: {es_remote_host} diff --git a/setup/master/.env.example b/setup/master/.env.example index 8c6bb6e..6bd384c 100644 --- a/setup/master/.env.example +++ b/setup/master/.env.example @@ -1,5 +1,8 @@ SPLUNK_PASSWORD=DFIR_passwd SPLUNK_LICENCE=Free +ELASTIC_PASSWORD=DFIR_passwd +ELASTIC_PORT=9200 +KIBANA_PORT=5601 RABBITMQ_USER=dfir RABBITMQ_PASSWORD=dfir POSTGRES_USER=dfir diff --git a/setup/master/docker-compose.yml b/setup/master/docker-compose.yml index 6b4226e..6d8a4fe 100755 --- a/setup/master/docker-compose.yml +++ b/setup/master/docker-compose.yml @@ -93,8 +93,8 @@ services: - SPLUNK_PASSWORD=${SPLUNK_PASSWORD} - SPLUNK_LICENSE_URI=${SPLUNK_LICENCE} ports: - - "8000:8000" # Splunk web interface - - "8089:8089" # Splunk services interface + - "${SPLUNK_PORT:-8000}:8000" # Splunk web interface + - "${SPLUNK_MPORT:-8089}:8089" # Splunk services interface - "9997:9997" # Event listening port - "8088:8088" # HEC port volumes: @@ -120,8 +120,8 @@ services: - SPLUNK_PASSWORD=${SPLUNK_PASSWORD} - SPLUNK_LICENSE_URI=${SPLUNK_LICENCE} ports: - - "8000:8000" # Splunk web interface - - "8089:8089" # Splunk services interface + - "${SPLUNK_PORT:-8000}:8000" # Splunk web interface + - "${SPLUNK_MPORT:-8089}:8089" # Splunk services interface - "9997:9997" # Event listening port - "8088:8088" # HEC port volumes: @@ -139,6 +139,90 @@ services: - "all" restart: always + elasticsearch: + build: + context: ./dockerfile + dockerfile: ElasticFile + container_name: master-elasticsearch + environment: + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD:-DFIR_passwd} + ports: + - "${ELASTIC_PORT:-9200}:9200" # Elasticsearch REST API + - "9300:9300" # Elasticsearch transport + volumes: + - ../../setup/elastic/data:/usr/share/elasticsearch/data + ulimits: + memlock: + soft: -1 + hard: -1 + healthcheck: + test: ["CMD-SHELL", "curl -s -f http://localhost:9200/_cluster/health | grep -q '\"status\":\"green\"\\|\"status\":\"yellow\"'"] + interval: 15s + timeout: 10s + retries: 60 + start_period: 60s + profiles: + - "elasticsearch-online" + - "all" + restart: always + + elasticsearch-offline: + image: master-elasticsearch + container_name: master-elasticsearch + environment: + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD:-DFIR_passwd} + ports: + - "${ELASTIC_PORT:-9200}:9200" # Elasticsearch REST API + - "9300:9300" # Elasticsearch transport + volumes: + - ../../setup/elastic/data:/usr/share/elasticsearch/data + ulimits: + memlock: + soft: -1 + hard: -1 + healthcheck: + test: ["CMD-SHELL", "curl -s -f http://localhost:9200/_cluster/health | grep -q '\"status\":\"green\"\\|\"status\":\"yellow\"'"] + interval: 15s + timeout: 10s + retries: 60 + start_period: 60s + profiles: + - "elasticsearch-offline" + - "all" + restart: always + + kibana: + build: + context: ./dockerfile + dockerfile: KibanaFile + container_name: master-kibana + environment: + - ELASTICSEARCH_SSL_VERIFICATIONMODE=none + ports: + - "${KIBANA_PORT:-5601}:5601" # Kibana web interface + depends_on: + elasticsearch: + condition: service_healthy + profiles: + - "elasticsearch-online" + - "all" + restart: always + + kibana-offline: + image: master-kibana + container_name: master-kibana + environment: + - ELASTICSEARCH_SSL_VERIFICATIONMODE=none + ports: + - "${KIBANA_PORT:-5601}:5601" # Kibana web interface + depends_on: + elasticsearch-offline: + condition: service_healthy + profiles: + - "elasticsearch-offline" + - "all" + restart: always + rabbitmq: build: dockerfile: RabbitmqFile diff --git a/setup/master/dockerfile/ElasticFile b/setup/master/dockerfile/ElasticFile new file mode 100644 index 0000000..427af90 --- /dev/null +++ b/setup/master/dockerfile/ElasticFile @@ -0,0 +1,22 @@ +FROM docker.elastic.co/elasticsearch/elasticsearch:9.4.3 + +# Single-node deployment. +ENV discovery.type=single-node \ + xpack.security.enabled=false \ + xpack.security.http.ssl.enabled=false \ + ES_JAVA_OPTS="-Xms8g -Xmx8g" \ + bootstrap.memory_lock=true + +# Enable CORS so Kibana / external dashboards can query from the browser if needed. +RUN echo 'http.cors.enabled: true' >> /usr/share/elasticsearch/config/elasticsearch.yml && \ + echo 'http.cors.allow-origin: "*"' >> /usr/share/elasticsearch/config/elasticsearch.yml + +# Indexing performance settings +RUN echo 'indices.memory.index_buffer_size: 512mb' >> /usr/share/elasticsearch/config/elasticsearch.yml && \ + echo 'thread_pool.write.queue_size: 10000' >> /usr/share/elasticsearch/config/elasticsearch.yml + +# Disable the disk watermark threshold so Elasticsearch keeps allocating +# shards instead of blocking indexing when free space drops below 90%. +RUN echo 'cluster.routing.allocation.disk.threshold_enabled: false' >> /usr/share/elasticsearch/config/elasticsearch.yml + +EXPOSE 9200 9300 \ No newline at end of file diff --git a/setup/master/dockerfile/KibanaFile b/setup/master/dockerfile/KibanaFile new file mode 100644 index 0000000..5d6cba7 --- /dev/null +++ b/setup/master/dockerfile/KibanaFile @@ -0,0 +1,13 @@ +FROM docker.elastic.co/kibana/kibana:9.4.3 + +# Point Kibana at the local Elasticsearch service. Security is disabled to match +# the local Elasticsearch container (xpack.security.enabled=false). +# Uses service name 'elasticsearch' (container: master-elasticsearch) +ENV ELASTICSEARCH_HOSTS=http://elasticsearch:9200 \ + SERVER_HOST=0.0.0.0 \ + SERVER_NAME=osir-kibana \ + XPACK_SECURITY_ENABLED=false \ + ELASTICSEARCH_SSL_VERIFICATIONMODE=none \ + ELASTICSEARCH_SSL_ALWAYSPRESENTCERTIFICATE=false + +EXPOSE 5601 diff --git a/setup/master/dockerfile/MasterFile b/setup/master/dockerfile/MasterFile index 8c7b4c3..05fee1f 100755 --- a/setup/master/dockerfile/MasterFile +++ b/setup/master/dockerfile/MasterFile @@ -4,7 +4,7 @@ FROM ubuntu:24.04 # - Environment -------------------------------- ENV DEBIAN_FRONTEND=noninteractive \ TZ=Europe/Minsk \ - PATH="/OSIR/OSIR:${PATH}" + PATH="/OSIR/OSIR:/OSIR/OSIR/bin:${PATH}" # - System setup ------------------------------- RUN set -eux; \ diff --git a/setup/master/dockerfile/sources/elastic/pipeline.json b/setup/master/dockerfile/sources/elastic/pipeline.json new file mode 100644 index 0000000..9ab0668 --- /dev/null +++ b/setup/master/dockerfile/sources/elastic/pipeline.json @@ -0,0 +1,29 @@ +{ + "id": "osir-timestamp", + "body": { + "description": "OSIR: promote the VRL 'timestamp' field to ElasticSearch '@timestamp'", + "processors": [ + { + "date": { + "field": "timestamp", + "target_field": "@timestamp", + "formats": ["ISO8601", "yyyy-MM-dd'T'HH:mm:ss'Z'", "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", "yyyy-MM-dd HH:mm:ss", "UNIX", "UNIX_MS"], + "on_failure": [ + { + "set": { + "field": "_ingest.inference_failure", + "value": "timestamp parse failed" + } + } + ] + } + }, + { + "set": { + "field": "event.ingested", + "value": "{{_ingest.timestamp}}" + } + } + ] + } +} diff --git a/setup/master/dockerfile/sources/elastic/template.json b/setup/master/dockerfile/sources/elastic/template.json new file mode 100644 index 0000000..20e18c1 --- /dev/null +++ b/setup/master/dockerfile/sources/elastic/template.json @@ -0,0 +1,92 @@ +{ + "id": "osir-template", + "body": { + "index_patterns": ["osir-*"], + "template": { + "settings": { + "number_of_shards": 1, + "number_of_replicas": 0, + "default_pipeline": "osir-timestamp", + "mapping.total_fields.limit": 20000, + "mapping.depth.limit": 50, + "mapping.nested_fields.limit": 500, + "index.mapping.ignore_malformed": true, + "index.mapping.coerce": true, + "index.refresh_interval": "-1", + "index.translog.durability": "async", + "index.translog.sync_interval": "30s", + "index.translog.flush_threshold_size": "1gb" + }, + "mappings": { + "dynamic": true, + "subobjects": false, + "properties": { + "@timestamp": { "type": "date" }, + "timestamp": { "type": "date" }, + "event": { + "properties": { + "original": { "type": "text" }, + "dataset": { "type": "keyword" }, + "kind": { "type": "keyword" }, + "category": { "type": "keyword" }, + "type": { "type": "keyword" }, + "action": { "type": "keyword" } + } + }, + "host": { + "properties": { + "name": { "type": "keyword" } + } + }, + "artifact": { "type": "keyword" }, + "sourcetype": { "type": "keyword" }, + "sourcefile": { "type": "keyword" }, + "case": { "type": "keyword" } + }, + "dynamic_templates": [ + { + "strings_as_keyword": { + "match_mapping_type": "string", + "mapping": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + { + "longs_as_long": { + "match_mapping_type": "long", + "mapping": { + "type": "long" + } + } + }, + { + "doubles_as_double": { + "match_mapping_type": "double", + "mapping": { + "type": "double" + } + } + }, + { + "booleans_as_boolean": { + "match_mapping_type": "boolean", + "mapping": { + "type": "boolean" + } + } + }, + { + "dates_as_date": { + "match_mapping_type": "date", + "mapping": { + "type": "date" + } + } + } + ] + } + } + } +} diff --git a/setup/master/master_setup.sh b/setup/master/master_setup.sh index 08f43f2..51b9c8b 100755 --- a/setup/master/master_setup.sh +++ b/setup/master/master_setup.sh @@ -28,6 +28,16 @@ config_mode=false # If set, nothing is ask to the user. Configuration is pulled offline_mode=false # OFFLINE MODE FLAG keep_splunk_data=true # If set, install Splunk without erasing previous data erase_splunk=false # If set, Splunk data is erased for new installation +keep_elastic_data=true # If set, install ElasticSearch without erasing previous data +erase_elastic=false # If set, ElasticSearch data is erased for new installation + +# Docker compose profiles are accumulated in two arrays so that base profiles +# and per-SIEM profiles can be built in any order without one clobbering the +# other (previously a single COMPOSE_PROFILES string was overwritten by +# build_base_profiles if called after the SIEM installers, which was a +# fragile, order-dependent pattern). +declare -a OSIR_BASE_PROFILES=() +declare -a OSIR_SIEM_PROFILES=() # Regular expression to match an IP address ip_regex='^([0-9]{1,3}\.){3}[0-9]{1,3}$' @@ -85,9 +95,13 @@ local_splunk_installation(){ $SETUP_SCRIPT_PATH/requirements.sh > /dev/null fi + # Read keep/erase flags from environment (set by install_from_conf or manual_install) + local keep_splunk_data="${KEEP_SPLUNK_DATA:-true}" + local erase_splunk="${ERASE_SPLUNK:-false}" + # If not a fresh install, erase previous Splunk data or stop installation - if ! $keep_splunk_data; then - if $erase_splunk; then + if [ "$keep_splunk_data" != "true" ]; then + if [ "$erase_splunk" = "true" ]; then # Erase Splunk data if $debug_mode; then $SETUP_SCRIPT_PATH/clean_splunk.sh @@ -100,39 +114,50 @@ local_splunk_installation(){ fi fi - # Launch docker - + # Append the splunk profile to the SIEM profile array (joined into + # COMPOSE_PROFILES by launch_docker_stack, order-independent). if $offline_mode; then - if is_wsl; then - export COMPOSE_PROFILES="default-offline,master-offline,splunk-offline" - else - export COMPOSE_PROFILES="default-offline,master-offline,splunk-offline,smb-offline" - fi + add_siem_profile "splunk-offline" else - if is_wsl; then - export COMPOSE_PROFILES="default,master-online,splunk-online" - else - export COMPOSE_PROFILES="default,master-online,splunk-online,smb-online" - fi + add_siem_profile "splunk-online" fi +} - export DOCKER_CONTAINERS=$(bash "$SETUP_SCRIPT_PATH/parse_docker_compose.sh" master) +remote_splunk_installation(){ + # Export to env requirements + export RAM_REQ="17000000" # ~16GB of ram + export DISK_REQ="150000" # 150GB of disk + + # Check and install requirements if $debug_mode; then - $SETUP_SCRIPT_PATH/setup_docker.sh master + $SETUP_SCRIPT_PATH/requirements.sh else - $SETUP_SCRIPT_PATH/setup_docker.sh master > /dev/null + $SETUP_SCRIPT_PATH/requirements.sh > /dev/null fi - if [ $? -eq 1 ]; then - (echo >&2 "${ERROR} Failed to launch Docker containers for local Splunk.") - exit 1 - fi + # Remote Splunk: no local container is launched. The base docker stack + # (master/rabbitmq/redis/...) is launched once by the caller. + + splunk_host=$1 + splunk_user=$2 + splunk_password=$3 + splunk_port=$4 + splunk_mport=$5 + splunk_ssl=$6 + + echo "splunk_host : $splunk_host" > /dev/null + echo "splunk_user : $splunk_user" > /dev/null + echo "splunk_password : $splunk_password" > /dev/null + echo "splunk_port : $splunk_port" > /dev/null + echo "splunk_mport : $splunk_mport" > /dev/null + echo "splunk_ssl : $splunk_ssl" > /dev/null } -remote_splunk_installation(){ + +local_es_installation(){ # Export to env requirements - export RAM_REQ="17000000" # ~16GB of ram - export DISK_REQ="150000" # 150GB of disk + export RAM_REQ="8000000" # ~8GB of ram + export DISK_REQ="100000" # 100GB of disk # Check and install requirements if $debug_mode; then @@ -141,24 +166,87 @@ remote_splunk_installation(){ $SETUP_SCRIPT_PATH/requirements.sh > /dev/null fi - # Launch docker + # Ensure elastic data directory exists with correct ownership for the + # elasticsearch container user (uid 1000) + ES_DATA_DIR=$(realpath "$MASTER_DIR/../../setup/elastic/data") + mkdir -p "$ES_DATA_DIR" + chown -R 1000:1000 "$ES_DATA_DIR" + + # Read keep/erase flags from environment (set by install_from_conf or manual_install) + local keep_es_data="${KEEP_ELASTIC_DATA:-true}" + local erase_es="${ERASE_ELASTIC:-false}" + + # If not a fresh install, erase previous ElasticSearch data or stop installation + if [ "$keep_es_data" != "true" ]; then + if [ "$erase_es" = "true" ]; then + # Erase ElasticSearch data + if $debug_mode; then + $SETUP_SCRIPT_PATH/clean_elastic.sh + else + $SETUP_SCRIPT_PATH/clean_elastic.sh > /dev/null + fi + else + (echo >&2 "${ERROR} Erase data ElasticSearch is not set, stopping execution.") + exit 0 + fi + fi + + # Append the elasticsearch profile to the SIEM profile array (joined into + # COMPOSE_PROFILES by launch_docker_stack, order-independent). + if $offline_mode; then + add_siem_profile "elasticsearch-offline" + else + add_siem_profile "elasticsearch-online" + fi +} + +remote_es_installation(){ + # No local container for remote ElasticSearch; nothing to launch here. + es_host=$1 + es_user=$2 + es_password=$3 + es_port=$4 + es_ssl=$5 + + echo "es_host : $es_host" > /dev/null + echo "es_user : $es_user" > /dev/null + echo "es_password : $es_password" > /dev/null + echo "es_port : $es_port" > /dev/null + echo "es_ssl : $es_ssl" > /dev/null +} - # OFFLINE check + +build_base_profiles(){ + # Build the common profiles array (master/rabbitmq/redis/samba) without any + # SIEM. Can be called before or after the SIEM installers: it only ever + # touches OSIR_BASE_PROFILES, never OSIR_SIEM_PROFILES. if $offline_mode; then if is_wsl; then - # Maybe no samba in WSL - export COMPOSE_PROFILES="default-offline,master-offline" + OSIR_BASE_PROFILES=("default-offline" "master-offline") else - export COMPOSE_PROFILES="default-offline,master-offline,smb-offline" + OSIR_BASE_PROFILES=("default-offline" "master-offline" "smb-offline") fi else if is_wsl; then - export COMPOSE_PROFILES="default,master-online" + OSIR_BASE_PROFILES=("default" "master-online") else - export COMPOSE_PROFILES="default,master-online,smb-online" + OSIR_BASE_PROFILES=("default" "master-online" "smb-online") fi fi +} + +add_siem_profile(){ + # Append a SIEM-specific docker compose profile (e.g. "splunk-online"). + # Kept separate from OSIR_BASE_PROFILES so build_base_profiles can be + # called in any order relative to the SIEM installers. + OSIR_SIEM_PROFILES+=("$1") +} +launch_docker_stack(){ + # Join the accumulated base + SIEM profiles into COMPOSE_PROFILES and + # launch the full docker stack once. + local all_profiles=("${OSIR_BASE_PROFILES[@]}" "${OSIR_SIEM_PROFILES[@]}") + export COMPOSE_PROFILES=$(IFS=,; echo "${all_profiles[*]}") export DOCKER_CONTAINERS=$(bash "$SETUP_SCRIPT_PATH/parse_docker_compose.sh" master) if $debug_mode; then @@ -166,170 +254,252 @@ remote_splunk_installation(){ else $SETUP_SCRIPT_PATH/setup_docker.sh master > /dev/null fi - # Check exit code if [ $? -eq 1 ]; then - (echo >&2 "${ERROR} Failed to launch docker requirements.") + (echo >&2 "${ERROR} Failed to launch Docker containers.") exit 1 fi - - splunk_host=$1 - splunk_user=$2 - splunk_password=$3 - splunk_port=$4 - splunk_mport=$5 - splunk_ssl=$6 - - echo "splunk_host : $splunk_host" > /dev/null - echo "splunk_user : $splunk_user" > /dev/null - echo "splunk_password : $splunk_password" > /dev/null - echo "splunk_port : $splunk_port" > /dev/null - echo "splunk_mport : $splunk_mport" > /dev/null - echo "splunk_ssl : $splunk_ssl" > /dev/null } install_from_conf(){ - # Get location of the windows box to determine installation type - splunk_location=$(get_yml_value "" splunk_location) # Error in yaml.sh if the key is first arg - if [ "$splunk_location" = "local" ] ; then - - # Check if Splunk was previously installed - if $debug_mode; then - $SETUP_SCRIPT_PATH/check_splunk.sh + # Determine which SIEM(s) to deploy + siem_selected=$(get_yml_value "" siem_selected) + if [ -z "$siem_selected" ]; then + siem_selected="splunk" + fi + + # Build the base (non-SIEM) docker profiles. Order relative to the + # SIEM installers below no longer matters (see build_base_profiles). + build_base_profiles + + # --- Splunk (if selected) --- + if [ "$siem_selected" = "splunk" ] || [ "$siem_selected" = "both" ]; then + splunk_location=$(get_yml_value "" splunk_location) + # Initialize keep/erase flags + keep_splunk_data=true + erase_splunk=false + if [ "$splunk_location" = "local" ] ; then + if $debug_mode; then + $SETUP_SCRIPT_PATH/check_splunk.sh + else + $SETUP_SCRIPT_PATH/check_splunk.sh > /dev/null + fi + if [ $? -eq 1 ]; then + keep_splunk_data=false + splunk_data=$(get_yml_value "" local_splunk_previous_data) + if [ "$splunk_data" = "keep" ] ; then + keep_splunk_data=true + elif [ "$splunk_data" = "erase" ] ; then + erase_splunk=true + fi + fi + export KEEP_SPLUNK_DATA=$keep_splunk_data + export ERASE_SPLUNK=$erase_splunk + local_splunk_installation + elif [ "$splunk_location" = "remote" ] ; then + splunk_host=$(get_yml_value "" splunk_remote_splunk_host) + splunk_user=$(get_yml_value "" splunk_user) + splunk_password=$(get_yml_value "" splunk_password) + splunk_port=$(get_yml_value "" splunk_port) + splunk_mport=$(get_yml_value "" splunk_mport) + splunk_ssl=$(get_yml_value "" splunk_ssl) + remote_splunk_installation $splunk_host $splunk_user $splunk_password $splunk_port $splunk_mport $splunk_ssl else - $SETUP_SCRIPT_PATH/check_splunk.sh > /dev/null + (echo >&2 "${ERROR} Wrong splunk location. Needs to be local or remote.") + exit 0 fi - # Check exit code: if enter condition if previous data found - if [ $? -eq 1 ]; then - keep_splunk_data=false - - # Get instruction if data present from previous installation - splunk_data=$(get_yml_value "" local_splunk_previous_data) - if [ "$splunk_data" = "keep" ] ; then - keep_splunk_data=true - elif [ "$splunk_data" = "erase" ] ; then - erase_splunk=true + fi + + # --- ElasticSearch (if selected) --- + if [ "$siem_selected" = "elasticsearch" ] || [ "$siem_selected" = "both" ]; then + es_location=$(get_yml_value "" elasticsearch_location) + # Initialize keep/erase flags + keep_elastic_data=true + erase_elastic=false + if [ "$es_location" = "local" ] ; then + if $debug_mode; then + $SETUP_SCRIPT_PATH/check_elastic.sh + else + $SETUP_SCRIPT_PATH/check_elastic.sh > /dev/null fi + if [ $? -eq 1 ]; then + keep_elastic_data=false + elastic_data=$(get_yml_value "" local_es_previous_data) + if [ "$elastic_data" = "keep" ] ; then + keep_elastic_data=true + elif [ "$elastic_data" = "erase" ] ; then + erase_elastic=true + fi + fi + export KEEP_ELASTIC_DATA=$keep_elastic_data + export ERASE_ELASTIC=$erase_elastic + local_es_installation + elif [ "$es_location" = "remote" ] ; then + es_host=$(get_yml_value "" elasticsearch_remote_es_host) + es_user=$(get_yml_value "" elasticsearch_user) + es_password=$(get_yml_value "" elasticsearch_password) + es_port=$(get_yml_value "" elasticsearch_port) + es_ssl=$(get_yml_value "" elasticsearch_ssl) + remote_es_installation $es_host $es_user $es_password $es_port $es_ssl + else + (echo >&2 "${ERROR} Wrong elasticsearch location. Needs to be local or remote.") + exit 0 fi - local_splunk_installation - elif [ "$splunk_location" = "remote" ] ; then - # Get Splunk host - splunk_host=$(get_yml_value "" splunk_remote_splunk_host) - # Get Splunk user - splunk_user=$(get_yml_value "" splunk_user) - # Get Splunk password - splunk_password=$(get_yml_value "" splunk_password) - # Get Splunk port - splunk_port=$(get_yml_value "" splunk_port) - # Get Splunk management port - splunk_mport=$(get_yml_value "" splunk_mport) - # Get Splunk ssl - splunk_ssl=$(get_yml_value "" splunk_ssl) - - # Start setup with a remote Splunk - remote_splunk_installation $splunk_host $splunk_user $splunk_password $splunk_port $splunk_mport $splunk_ssl - else - (echo >&2 "${ERROR} Wrong location. Needs to be local or remote.") - exit 0 + kibana_port=$(get_yml_value "" kibana_port) fi + +# Persist SIEM choice and ES values for the agent/master config + export SIEM_SELECTED=$siem_selected + export ELASTIC_LOCATION=${es_location:-remote} + export ELASTIC_HOST=${es_host:-127.0.0.1} + export ELASTIC_PORT=${es_port:-9200} + export ELASTIC_USER=${es_user:-elastic} + export ELASTIC_PASSWORD=${es_password:-DFIR_passwd} + export ELASTIC_SSL=${es_ssl:-False} + export ELASTIC_REMOTE_HOST=${es_host:-127.0.0.1} + export KIBANA_PORT=${kibana_port:-5601} + export KEEP_ELASTIC_DATA=$keep_elastic_data + export ERASE_ELASTIC=$erase_elastic + export KEEP_SPLUNK_DATA=$keep_splunk_data + export ERASE_SPLUNK=$erase_splunk + + # Launch the accumulated stack (base + SIEM profiles already appended) + launch_docker_stack } manual_install(){ - #local_installation - - # Ask user : local or remote splunk - default_location="local" - read -p "$(echo -n >&2 "${USERINPUT} Do you want to setup a local Splunk server or configure a remote one ? [Default is: $default_location] [options: local/remote]: ")" splunk_location - - # Ask user : remote Splunk user - default_user="admin" - read -p "$(echo -n >&2 "${USERINPUT} Enter the Splunk user for administration and event forwarding (need to be admin). [Default is: $default_user]: ")" splunk_user - if [[ -z "$splunk_user" ]]; then - splunk_user="$default_user" + # 1) Choose which SIEM to deploy + default_siem="splunk" + read -p "$(echo -n >&2 "${USERINPUT} Which SIEM do you want to use for visualization ? [Default is: $default_siem] [options: splunk/elasticsearch/both]: ")" siem_selected + if [[ -z "$siem_selected" ]]; then + siem_selected="$default_siem" fi - - # Ask user : remote Splunk password - default_password="DFIR_passwd" - read -p "$(echo -n >&2 "${USERINPUT} Enter the Splunk password. [Default is: $default_password]: ")" splunk_password - if [[ -z "$splunk_password" ]]; then - splunk_password="$default_password" + siem_selected=$(echo "$siem_selected" | tr '[:upper:]' '[:lower:]') + if [[ "$siem_selected" != "splunk" && "$siem_selected" != "elasticsearch" && "$siem_selected" != "both" ]]; then + (echo >&2 "${ERROR} Invalid SIEM choice. Use splunk, elasticsearch or both.") + exit 0 fi - # Ask user : remote Splunk port - default_splunk_port="8000" - read -p "$(echo -n >&2 "${USERINPUT} Enter the Splunk port. [Default is: $default_splunk_port]: ")" splunk_port - if [[ -z "$splunk_port" ]]; then - splunk_port="$default_splunk_port" - fi + # Build the base (non-SIEM) docker profiles. Order relative to the + # SIEM installers below no longer matters (see build_base_profiles). + build_base_profiles - # Ask user : remote Splunk management port - default_splunk_mport="8089" - read -p "$(echo -n >&2 "${USERINPUT} Enter the Splunk management port. [Default is: $default_splunk_mport]: ")" splunk_mport - if [[ -z "$splunk_mport" ]]; then - splunk_mport="$default_splunk_mport" - fi + # 2) Configure Splunk if selected + if [ "$siem_selected" = "splunk" ] || [ "$siem_selected" = "both" ]; then + read -p "$(echo -n >&2 "${USERINPUT} Setup a local Splunk server or configure a remote one ? [Default is: local] [options: local/remote]: ")" splunk_location + if [[ -z "$splunk_location" ]]; then splunk_location="local"; fi - # Ask user : remote Splunk management port - default_splunk_ssl="False" - read -p "$(echo -n >&2 "${USERINPUT} Enable SSL for Splunk communication ? . [Default is: $default_splunk_ssl] [options: True/False]: ")" splunk_ssl - if [[ -z "$splunk_ssl" ]]; then - splunk_ssl="$default_splunk_ssl" - fi + default_user="admin" + read -p "$(echo -n >&2 "${USERINPUT} Enter the Splunk user (need to be admin). [Default is: $default_user]: ")" splunk_user + if [[ -z "$splunk_user" ]]; then splunk_user="$default_user"; fi + + default_password="DFIR_passwd" + read -p "$(echo -n >&2 "${USERINPUT} Enter the Splunk password. [Default is: $default_password]: ")" splunk_password + if [[ -z "$splunk_password" ]]; then splunk_password="$default_password"; fi + + default_splunk_port="8000" + read -p "$(echo -n >&2 "${USERINPUT} Enter the Splunk web port. [Default is: $default_splunk_port]: ")" splunk_port + if [[ -z "$splunk_port" ]]; then splunk_port="$default_splunk_port"; fi + + default_splunk_mport="8089" + read -p "$(echo -n >&2 "${USERINPUT} Enter the Splunk management port. [Default is: $default_splunk_mport]: ")" splunk_mport + if [[ -z "$splunk_mport" ]]; then splunk_mport="$default_splunk_mport"; fi + + default_splunk_ssl="False" + read -p "$(echo -n >&2 "${USERINPUT} Enable SSL for Splunk communication ? [Default is: $default_splunk_ssl] [options: True/False]: ")" splunk_ssl + if [[ -z "$splunk_ssl" ]]; then splunk_ssl="$default_splunk_ssl"; fi - # Setup local Splunk server - if [ -z "$splunk_location" ] || [ "$splunk_location" = "local" ] ; then - splunk_location="local" - splunk_host="127.0.0.1" - - # Check if Splunk was previously installed - if $debug_mode; then - $SETUP_SCRIPT_PATH/check_splunk.sh + if [ "$splunk_location" = "local" ] ; then + splunk_host="127.0.0.1" + # Initialize keep/erase flags + keep_splunk_data=true + erase_splunk=false + if $debug_mode; then + $SETUP_SCRIPT_PATH/check_splunk.sh + else + $SETUP_SCRIPT_PATH/check_splunk.sh > /dev/null + fi + if [ $? -eq 1 ]; then + keep_splunk_data=false + default_choice=stop + read -p "$(echo -n >&2 "${USERINPUT} Splunk data found. continue (keep) / erase / stop ? [Default is: $default_choice] [options: stop/continue/erase]: ")" user_choice + if [[ -z "$user_choice" ]]; then user_choice="$default_choice"; fi + if [ "$user_choice" = "continue" ]; then keep_splunk_data=true; elif [ "$user_choice" = "erase" ]; then erase_splunk=true; fi + fi + export KEEP_SPLUNK_DATA=$keep_splunk_data + export ERASE_SPLUNK=$erase_splunk + local_splunk_installation else - $SETUP_SCRIPT_PATH/check_splunk.sh > /dev/null - fi - # Check exit code: if enter condition if previous data found - if [ $? -eq 1 ]; then - keep_splunk_data=false - # Ask user : proceed Splunk installation and erase files or stop installation - default_choice=stop - read -p "$(echo -n >&2 "${USERINPUT} To continue Splunk installation, select continue if you want to restart the same instance or erase for fresh new install. [Default is: $default_choice] [options: stop/continue/erase]: ")" user_choice - if [[ -z "$user_choice" ]]; then - user_choice="$default_choice" - - elif [ "$user_choice" = "continue" ]; then - keep_splunk_data=true - - elif [ "$user_choice" = "erase" ]; then - erase_splunk=true + default_splunk_host="host.docker.internal" + read -p "$(echo -n >&2 "${USERINPUT} Enter the Splunk host. [Default is: $default_splunk_host] [options: IP/FQDN]: ")" splunk_host + if [[ -z "$splunk_host" ]]; then splunk_host="$default_splunk_host"; fi + if [[ ! $splunk_host =~ $ip_regex && ! $splunk_host =~ $fqdn_regex ]]; then + (echo >&2 "${ERROR} Please enter valid IP or FQDN.") + exit 0 fi + remote_splunk_installation $splunk_host $splunk_user $splunk_password $splunk_port $splunk_mport $splunk_ssl fi - local_splunk_installation $splunk_user $splunk_password $splunk_port $splunk_mport $splunk_ssl - - elif [ "$splunk_location" = "remote" ] ; then - # Ask user : remote Splunk host IP/FQDN - default_splunk_host="host.docker.internal" - read -p "$(echo -n >&2 "${USERINPUT} Enter the Splunk host. [Default is: $default_splunk_host] [options: IP/FQDN]: ")" splunk_host - - # Check if the variable contains an IP address or a FQDN - if [[ -z "$splunk_host" ]]; then - splunk_host="$default_splunk_host" - fi - if [[ $splunk_host =~ $ip_regex || $splunk_host =~ $fqdn_regex ]]; then - (echo >&2 "${INFO} Valid host string.") + fi + + # 3) Configure ElasticSearch if selected + if [ "$siem_selected" = "elasticsearch" ] || [ "$siem_selected" = "both" ]; then + read -p "$(echo -n >&2 "${USERINPUT} Setup a local ElasticSearch server or configure a remote one ? [Default is: local] [options: local/remote]: ")" es_location + if [[ -z "$es_location" ]]; then es_location="local"; fi + + default_es_user="elastic" + read -p "$(echo -n >&2 "${USERINPUT} Enter the ElasticSearch user. [Default is: $default_es_user]: ")" es_user + if [[ -z "$es_user" ]]; then es_user="$default_es_user"; fi + + default_es_password="DFIR_passwd" + read -p "$(echo -n >&2 "${USERINPUT} Enter the ElasticSearch password. [Default is: $default_es_password]: ")" es_password + if [[ -z "$es_password" ]]; then es_password="$default_es_password"; fi + + default_es_port="9200" + read -p "$(echo -n >&2 "${USERINPUT} Enter the ElasticSearch REST port. [Default is: $default_es_port]: ")" es_port + if [[ -z "$es_port" ]]; then es_port="$default_es_port"; fi + + default_es_ssl="False" + read -p "$(echo -n >&2 "${USERINPUT} Enable SSL for ElasticSearch communication ? [Default is: $default_es_ssl] [options: True/False]: ")" es_ssl + if [[ -z "$es_ssl" ]]; then es_ssl="$default_es_ssl"; fi + + default_kibana_port="5601" + read -p "$(echo -n >&2 "${USERINPUT} Enter the Kibana web port. [Default is: $default_kibana_port]: ")" kibana_port + if [[ -z "$kibana_port" ]]; then kibana_port="$default_kibana_port"; fi + + if [ "$es_location" = "local" ] ; then + es_host="127.0.0.1" + # Initialize keep/erase flags + keep_elastic_data=true + erase_elastic=false + if $debug_mode; then + $SETUP_SCRIPT_PATH/check_elastic.sh + else + $SETUP_SCRIPT_PATH/check_elastic.sh > /dev/null + fi + if [ $? -eq 1 ]; then + keep_elastic_data=false + default_choice=stop + read -p "$(echo -n >&2 "${USERINPUT} ElasticSearch data found. continue (keep) / erase / stop ? [Default is: $default_choice] [options: stop/continue/erase]: ")" user_choice + if [[ -z "$user_choice" ]]; then user_choice="$default_choice"; fi + if [ "$user_choice" = "continue" ]; then keep_elastic_data=true; elif [ "$user_choice" = "erase" ]; then erase_elastic=true; fi + fi + export KEEP_ELASTIC_DATA=$keep_elastic_data + export ERASE_ELASTIC=$erase_elastic + local_es_installation else - echo "splunk_host : $splunk_host" - (echo >&2 "${ERROR} Please enter valid IP or FQDN.") - exit 0 + default_es_host="host.docker.internal" + read -p "$(echo -n >&2 "${USERINPUT} Enter the ElasticSearch host. [Default is: $default_es_host] [options: IP/FQDN]: ")" es_host + if [[ -z "$es_host" ]]; then es_host="$default_es_host"; fi + if [[ ! $es_host =~ $ip_regex && ! $es_host =~ $fqdn_regex ]]; then + (echo >&2 "${ERROR} Please enter valid IP or FQDN.") + exit 0 + fi + remote_es_installation $es_host $es_user $es_password $es_port $es_ssl fi - - # Start setup with remote splunk server - remote_splunk_installation $splunk_host $splunk_user $splunk_password $splunk_port $splunk_mport $splunk_ssl - else - (echo >&2 "${ERROR} Please select local or remote") - exit 0 fi - # Export users input to env + # 4) Export users input to env + export SIEM_SELECTED=$siem_selected export SPLUNK_LOCATION=$splunk_location export SPLUNK_USER=$splunk_user export SPLUNK_PASSWORD=$splunk_password @@ -337,8 +507,48 @@ manual_install(){ export SPLUNK_PORT=$splunk_port export SPLUNK_MPORT=$splunk_mport export SPLUNK_SSL=$splunk_ssl + export ELASTIC_LOCATION=$es_location + export ELASTIC_HOST=$es_host + export ELASTIC_USER=$es_user + export ELASTIC_PASSWORD=$es_password + export ELASTIC_PORT=$es_port + export ELASTIC_SSL=$es_ssl + export ELASTIC_REMOTE_HOST=$es_host + export KIBANA_PORT=${kibana_port:-5601} + + # Persist previous data choice for local SIEMs + if [ "$splunk_location" = "local" ]; then + if $keep_splunk_data; then + export LOCAL_SPLUNK_PREVIOUS_DATA="keep" + elif $erase_splunk; then + export LOCAL_SPLUNK_PREVIOUS_DATA="erase" + else + export LOCAL_SPLUNK_PREVIOUS_DATA="stop" + fi + fi + + if [ "$es_location" = "local" ]; then + if $keep_elastic_data; then + export LOCAL_ELASTIC_PREVIOUS_DATA="keep" + elif $erase_elastic; then + export LOCAL_ELASTIC_PREVIOUS_DATA="erase" + else + export LOCAL_ELASTIC_PREVIOUS_DATA="stop" + fi + + export KEEP_ELASTIC_DATA=$keep_elastic_data + export ERASE_ELASTIC=$erase_elastic + fi + + if [ "$splunk_location" = "local" ]; then + export KEEP_SPLUNK_DATA=$keep_splunk_data + export ERASE_SPLUNK=$erase_splunk + fi + + # 5) Launch the accumulated stack (base + SIEM profiles already appended) + launch_docker_stack - # Save setup config + # 6) Save setup config conf_master_sample="$CONF_PATH/master_sample.yml" conf_master="$CONF_PATH/master.yml" # CHANGE ME - temporary for dev if $debug_mode; then diff --git a/setup/setup_scripts/check_elastic.sh b/setup/setup_scripts/check_elastic.sh new file mode 100755 index 0000000..2bbc5ca --- /dev/null +++ b/setup/setup_scripts/check_elastic.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +ERROR=$(tput setaf 1; echo -n " [!]"; tput sgr0) +GOODTOGO=$(tput setaf 2; echo -n " [✓]"; tput sgr0) +INFO=$(tput setaf 3; echo -n " [-]"; tput sgr0) +USERINPUT=$(tput setaf 4; echo -n " [?]"; tput sgr0) +MASTER_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) + +elastic_path="$MASTER_DIR/../../setup/elastic/data/" +check_elastic_install(){ + if [ -d "$elastic_path" ]; then + file_count=$(find "$elastic_path" -type f | wc -l) + else + file_count=0 + fi + # Check if the file count is greater than zero + if [ "$file_count" -gt 0 ]; then + (echo >&2 "${INFO} $elastic_path contains files, ElasticSearch was previously installed.") + exit 1 + else + (echo >&2 "${INFO} $elastic_path does not contain file, ElasticSearch can be installed.") + fi +} + +main() { + check_elastic_install +} +main \ No newline at end of file diff --git a/setup/setup_scripts/clean_elastic.sh b/setup/setup_scripts/clean_elastic.sh new file mode 100755 index 0000000..b095685 --- /dev/null +++ b/setup/setup_scripts/clean_elastic.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +ERROR=$(tput setaf 1; echo -n " [!]"; tput sgr0) +GOODTOGO=$(tput setaf 2; echo -n " [✓]"; tput sgr0) +INFO=$(tput setaf 3; echo -n " [-]"; tput sgr0) +USERINPUT=$(tput setaf 4; echo -n " [?]"; tput sgr0) +MASTER_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) + +elastic_data="$MASTER_DIR/../../setup/elastic/data/*/*" +clean_elastic_data(){ + rm -rf $elastic_data + (echo >&2 "${INFO} files in $elastic_data erased.") + # Ensure the data directory exists with correct ownership for the ES container. + ES_DIR=$(realpath "$MASTER_DIR/../../setup/elastic/data") + mkdir -p "$ES_DIR" + chown -R 1000:1000 "$ES_DIR" +} + +main() { + clean_elastic_data +} +main \ No newline at end of file diff --git a/setup/setup_scripts/save_setup_config.sh b/setup/setup_scripts/save_setup_config.sh index 07e46b5..28acb1f 100755 --- a/setup/setup_scripts/save_setup_config.sh +++ b/setup/setup_scripts/save_setup_config.sh @@ -20,6 +20,12 @@ save_agent_setup_conf() { sed -i "s/{splunk_port}/$SPLUNK_PORT/g" $conf sed -i "s/{splunk_mport}/$SPLUNK_MPORT/g" $conf sed -i "s/{splunk_ssl}/$SPLUNK_SSL/g" $conf + sed -i "s/{es_host}/$ELASTIC_HOST/g" $conf + sed -i "s/{es_user}/$ELASTIC_USER/g" $conf + sed -i "s/{es_password}/$ELASTIC_PASSWORD/g" $conf + sed -i "s/{es_port}/$ELASTIC_PORT/g" $conf + sed -i "s/{es_ssl}/$ELASTIC_SSL/g" $conf + sed -i "s/{kibana_port}/${KIBANA_PORT:-5601}/g" $conf # Return the exit status of the last command executed return $? @@ -27,6 +33,7 @@ save_agent_setup_conf() { save_master_setup_conf() { cp $conf_sample $conf + sed -i "s/{siem_selected}/$SIEM_SELECTED/g" $conf sed -i "s/{splunk_location}/$SPLUNK_LOCATION/g" $conf sed -i "s/{splunk_user}/$SPLUNK_USER/g" $conf sed -i "s/{splunk_password}/$SPLUNK_PASSWORD/g" $conf @@ -34,7 +41,17 @@ save_master_setup_conf() { sed -i "s/{splunk_port}/$SPLUNK_PORT/g" $conf sed -i "s/{splunk_mport}/$SPLUNK_MPORT/g" $conf sed -i "s/{splunk_ssl}/$SPLUNK_SSL/g" $conf - + sed -i "s/{es_location}/$ELASTIC_LOCATION/g" $conf + sed -i "s/{es_host}/$ELASTIC_HOST/g" $conf + sed -i "s/{es_port}/$ELASTIC_PORT/g" $conf + sed -i "s/{es_user}/$ELASTIC_USER/g" $conf + sed -i "s/{es_password}/$ELASTIC_PASSWORD/g" $conf + sed -i "s/{es_ssl}/$ELASTIC_SSL/g" $conf + sed -i "s/{es_remote_host}/$ELASTIC_REMOTE_HOST/g" $conf + sed -i "s/{kibana_port}/${KIBANA_PORT:-5601}/g" $conf + sed -i "s/{local_elastic_previous_data}/$LOCAL_ELASTIC_PREVIOUS_DATA/g" $conf + sed -i "s/{local_splunk_previous_data}/$LOCAL_SPLUNK_PREVIOUS_DATA/g" $conf + # Return the exit status of the last command executed return $? } diff --git a/setup/setup_scripts/setup_docker.sh b/setup/setup_scripts/setup_docker.sh index 924a522..8d62c05 100755 --- a/setup/setup_scripts/setup_docker.sh +++ b/setup/setup_scripts/setup_docker.sh @@ -37,11 +37,25 @@ start_docker_compose() { set_env_var "HOST_IP_LIST" "$(hostname -I | tr ' ' ',')" set_env_var "WINDOWS_CORES" "$WINDOWS_CORES" + # Persist user-selected SIEM ports so docker compose (run as root via sudo, + # which does not inherit the exported shell variables) publishes the same + # ports the user was asked for during setup. + [ -n "$SPLUNK_PORT" ] && set_env_var "SPLUNK_PORT" "$SPLUNK_PORT" + [ -n "$SPLUNK_MPORT" ] && set_env_var "SPLUNK_MPORT" "$SPLUNK_MPORT" + [ -n "$ELASTIC_PORT" ] && set_env_var "ELASTIC_PORT" "$ELASTIC_PORT" + [ -n "$KIBANA_PORT" ] && set_env_var "KIBANA_PORT" "$KIBANA_PORT" + if is_wsl; then set_env_var "WSL_INTEROP" "$WSL_INTEROP" set_env_var "OSIR_PATH" "$(wslpath -w "$MASTER_DIR/../../../")" fi + # Ensure elasticsearch data directory exists with correct ownership (uid 1000) + # before starting containers, so the bind mount is writable by the container. + local es_data_dir="$DOCKER_COMPOSE_REPO/../../setup/elastic/data" + sudo mkdir -p "$es_data_dir" + sudo chown -R 1000:1000 "$es_data_dir" + if [ -n "$COMPOSE_PROFILES" ]; then sudo COMPOSE_PROFILES="$COMPOSE_PROFILES" docker compose -f "$DOCKER_COMPOSE_REPO/docker-compose.yml" up -d else