output/reference: Include reference information in alert (if configured)

doc/userguide/output/eve/eve-json-output.rst

@@ -89,6 +89,9 @@ Metadata::
                 # Log the raw rule text.
                 #raw: false
 
+                # Include the rule reference information
+                #reference: false
+
 Anomaly
 ~~~~~~~
 

doc/userguide/partials/eve-log.yaml

@@ -75,6 +75,19 @@ outputs:
             # payload-length: yes      # enable dumping payload length, including the gaps
             # packet: yes              # enable dumping of packet (without stream segments)
             # metadata: no             # enable inclusion of app layer metadata with alert. Default yes
+            # If you want metadata, use:
+            # metadata:
+              # Include the decoded application layer (ie. http, dns)
+              #app-layer: true
+              # Log the current state of the flow record.
+              #flow: true
+              #rule:
+                # Log the metadata field from the rule in a structured
+                # format.
+                #metadata: true
+                # Log the raw rule text.
+                #raw: false
+                #reference: false      # include reference information from the rule
             # http-body: yes           # Requires metadata; enable dumping of HTTP body in Base64
             # http-body-printable: yes # Requires metadata; enable dumping of HTTP body in printable format
             # websocket-payload: yes   # Requires metadata; enable dumping of WebSocket Payload in Base64

etc/reference.config

@@ -1,26 +1,44 @@
 # config reference: system URL
 
-config reference: bugtraq   http://www.securityfocus.com/bid/
-config reference: bid	    http://www.securityfocus.com/bid/
-config reference: cve       http://cve.mitre.org/cgi-bin/cvename.cgi?name=
-#config reference: cve       http://cvedetails.com/cve/
-config reference: secunia   http://www.secunia.com/advisories/
+#
+# Note: https// used
+##############################
+# Referenced by ET/Open ET/Pro
+##############################
+
+#  resolves, works as intended
+config reference: cve       https://cve.mitre.org/cgi-bin/cvename.cgi?name=
+config reference: nessus    https://www.tenable.com/plugins/nessus/
+config reference: url       https://
 
 #whitehats is unfortunately gone
-config reference: arachNIDS http://www.whitehats.com/info/IDS
-
-config reference: McAfee    http://vil.nai.com/vil/content/v_
-config reference: nessus    http://cgi.nessus.org/plugins/dump.php3?id=
-config reference: url       http://
-config reference: et        http://doc.emergingthreats.net/
-config reference: etpro     http://doc.emergingthreatspro.com/
-config reference: telus     http://
+#
+#  no longer resolves
+config reference: McAfee    https://vil.nai.com/vil/content/v_
+config reference: bid	    https://www.securityfocus.com/bid/
+config reference: bugtraq   https://www.securityfocus.com/bid/
+config reference: md5	    https://www.threatexpert.com/report.aspx?md5=
+
+#  resolves, but non-useful page
+config reference: secunia   https://www.secunia.com/advisories/
+config reference: arachNIDS https://www.whitehats.com/info/IDS
+
+###################################################
+# No longer referenced from ET/Open ET/Pro rulesets
+###################################################
+
+#  resolves
+config reference: exploitdb https://www.exploit-db.com/exploits/
+config reference: msft      https://technet.microsoft.com/security/bulletin/
+
+#  resolves, but non-useful page
+config reference: et        https://doc.emergingthreats.net/
+config reference: etpro     https://doc.emergingthreatspro.com/
+config reference: telus     https://
+
+#  no longer resolves
+config reference: xforce    http://xforce.iss.net/xforce/xfdb/
 config reference: osvdb     http://osvdb.org/show/osvdb/
 config reference: threatexpert http://www.threatexpert.com/report.aspx?md5=
-config reference: md5	    http://www.threatexpert.com/report.aspx?md5=
-config reference: exploitdb http://www.exploit-db.com/exploits/
 config reference: openpacket https://www.openpacket.org/capture/grab/
 config reference: securitytracker http://securitytracker.com/id?
-config reference: secunia   http://secunia.com/advisories/
-config reference: xforce    http://xforce.iss.net/xforce/xfdb/
-config reference: msft      http://technet.microsoft.com/security/bulletin/

etc/schema.json

@@ -288,6 +288,13 @@
                     },
                     "additionalProperties": true
                 },
+                "references": {
+                    "type": "array",
+                    "minItems": 1,
+                    "items": {
+                        "type": "string"
+                    }
+                },
                 "source": {
                     "type": "object",
                     "properties": {

src/detect-reference.c

@@ -43,7 +43,10 @@
 #include "util-byte.h"
 #include "util-debug.h"
 
-#define PARSE_REGEX "^\\s*([A-Za-z0-9]+)\\s*,\"?\\s*\"?\\s*([a-zA-Z0-9\\-_\\.\\/\\?\\=]+)\"?\\s*\"?"
+/* Breakout key and scheme (optional) and domain/path (mandatory) */
+#define PARSE_REGEX                                                                                \
+    "^\\s*([A-Za-z0-9]+)\\s*,\"?\\s*\"?\\s*([a-zA-Z]+:\\/\\/)?([a-zA-Z0-9\\-_\\.\\/"               \
+    "\\?\\=]+)\"?\\s*\"?"
 
 static DetectParseRegex parse_regex;
 
@@ -74,6 +77,9 @@ void DetectReferenceFree(DetectReference *ref)
 {
     SCEnter();
 
+    if (ref->key)
+        SCFree(ref->key);
+
     if (ref->reference != NULL) {
         SCFree(ref->reference);
     }
@@ -98,11 +104,12 @@ static DetectReference *DetectReferenceParse(const char *rawstr, DetectEngineCtx
     int res = 0;
     size_t pcre2len;
     char key[REFERENCE_SYSTEM_NAME_MAX] = "";
-    char content[REFERENCE_CONTENT_NAME_MAX] = "";
+    char scheme[REFERENCE_SYSTEM_NAME_MAX] = "";
+    char uri[REFERENCE_CONTENT_NAME_MAX] = "";
 
     pcre2_match_data *match = NULL;
     int ret = DetectParsePcreExec(&parse_regex, &match, rawstr, 0, 0);
-    if (ret < 2) {
+    if (ret != 4) {
         SCLogError("Unable to parse \"reference\" "
                    "keyword argument - \"%s\".   Invalid argument.",
                 rawstr);
@@ -118,51 +125,72 @@ static DetectReference *DetectReferenceParse(const char *rawstr, DetectEngineCtx
         return NULL;
     }
 
+    /* Position 1 = key (mandatory) */
     pcre2len = sizeof(key);
     res = pcre2_substring_copy_bynumber(match, 1, (PCRE2_UCHAR8 *)key, &pcre2len);
     if (res < 0) {
-        SCLogError("pcre2_substring_copy_bynumber failed");
+        SCLogError("pcre2_substring_copy_bynumber key failed");
         goto error;
     }
 
-    pcre2len = sizeof(content);
-    res = pcre2_substring_copy_bynumber(match, 2, (PCRE2_UCHAR8 *)content, &pcre2len);
+    /* Position 2 = scheme (optional) */
+    pcre2len = sizeof(scheme);
+    (void)pcre2_substring_copy_bynumber(match, 2, (PCRE2_UCHAR8 *)scheme, &pcre2len);
+
+    /* Position 3 = domain-path (mandatory) */
+    pcre2len = sizeof(uri);
+    res = pcre2_substring_copy_bynumber(match, 3, (PCRE2_UCHAR8 *)uri, &pcre2len);
     if (res < 0) {
-        SCLogError("pcre2_substring_copy_bynumber failed");
+        SCLogError("pcre2_substring_copy_bynumber domain-path failed");
         goto error;
     }
 
-    if (strlen(key) == 0 || strlen(content) == 0)
+    int ref_len = strlen(uri);
+    /* no key, reference -- return an error */
+    if (strlen(key) == 0 || ref_len == 0)
         goto error;
 
-    SCRConfReference *lookup_ref_conf = SCRConfGetReference(key, de_ctx);
-    if (lookup_ref_conf != NULL) {
-        ref->key = lookup_ref_conf->url;
+    if (strlen(scheme)) {
+        SCLogConfig("scheme value %s overrides key %s", scheme, key);
+        ref->key = SCStrdup(scheme);
+        /* already bound checked to be REFERENCE_SYSTEM_NAME_MAX or less */
+        ref->key_len = (uint16_t)strlen(scheme);
     } else {
-        if (SigMatchStrictEnabled(DETECT_REFERENCE)) {
-            SCLogError("unknown reference key \"%s\"", key);
-            goto error;
-        }
-
-        SCLogWarning("unknown reference key \"%s\"", key);
 
-        char str[2048];
-        snprintf(str, sizeof(str), "config reference: %s undefined\n", key);
-
-        if (SCRConfAddReference(de_ctx, str) < 0)
-            goto error;
-        lookup_ref_conf = SCRConfGetReference(key, de_ctx);
-        if (lookup_ref_conf == NULL)
-            goto error;
+        SCRConfReference *lookup_ref_conf = SCRConfGetReference(key, de_ctx);
+        if (lookup_ref_conf != NULL) {
+            ref->key = SCStrdup(lookup_ref_conf->url);
+            /* already bound checked to be REFERENCE_SYSTEM_NAME_MAX or less */
+            ref->key_len = (uint16_t)strlen(ref->key);
+        } else {
+            if (SigMatchStrictEnabled(DETECT_REFERENCE)) {
+                SCLogError("unknown reference key \"%s\"", key);
+                goto error;
+            }
+
+            SCLogWarning("unknown reference key \"%s\"", key);
+
+            char str[2048];
+            snprintf(str, sizeof(str), "config reference: %s undefined\n", key);
+
+            if (SCRConfAddReference(de_ctx, str) < 0)
+                goto error;
+            lookup_ref_conf = SCRConfGetReference(key, de_ctx);
+            if (lookup_ref_conf == NULL)
+                goto error;
+        }
     }
 
     /* make a copy so we can free pcre's substring */
-    ref->reference = SCStrdup(content);
+    ref->reference = SCStrdup(uri);
     if (ref->reference == NULL) {
         SCLogError("strdup failed: %s", strerror(errno));
         goto error;
     }
 
+    /* already bound checked to be REFERENCE_CONTENT_NAME_MAX or less */
+    ref->reference_len = (uint16_t)ref_len;
+
     pcre2_match_data_free(match);
     /* free the substrings */
     SCReturnPtr(ref, "Reference");

src/detect-reference.h

@@ -1,4 +1,4 @@
-/* Copyright (C) 2007-2010 Open Information Security Foundation
+/* Copyright (C) 2007-2024 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -32,6 +32,13 @@ typedef struct DetectReference_ {
     char *key;
     /* reference data */
     char *reference;
+
+    /*
+     * These have been length checked against REFERENCE_SYSTEM_NAME_MAX,
+     * and REFERENCE_CONTENT_NAME_MAX
+     */
+    uint16_t key_len;
+    uint16_t reference_len;
     /* next reference in the signature */
     struct DetectReference_ *next;
 } DetectReference;

src/output-json-alert.c

@@ -1,4 +1,4 @@
-/* Copyright (C) 2013-2023 Open Information Security Foundation
+/* Copyright (C) 2013-2024 Open Information Security Foundation
  *
  * You can copy, redistribute or modify this Program under the terms of
  * the GNU General Public License version 2 as published by the Free
@@ -63,6 +63,7 @@
 #include "util-print.h"
 #include "util-optimize.h"
 #include "util-buffer.h"
+#include "util-reference-config.h"
 #include "util-validate.h"
 
 #include "action-globals.h"
@@ -83,6 +84,7 @@
 #define LOG_JSON_WEBSOCKET_PAYLOAD        BIT_U16(11)
 #define LOG_JSON_WEBSOCKET_PAYLOAD_BASE64 BIT_U16(12)
 #define LOG_JSON_PAYLOAD_LENGTH           BIT_U16(13)
+#define LOG_JSON_REFERENCE                BIT_U16(14)
 
 #define METADATA_DEFAULTS ( LOG_JSON_FLOW |                        \
             LOG_JSON_APP_LAYER  |                                  \
@@ -169,18 +171,37 @@ static void AlertJsonSourceTarget(const Packet *p, const PacketAlert *pa,
     jb_close(js);
 }
 
-static void AlertJsonMetadata(AlertJsonOutputCtx *json_output_ctx,
-        const PacketAlert *pa, JsonBuilder *js)
+static void AlertJsonReference(const PacketAlert *pa, JsonBuilder *jb)
+{
+    if (!pa->s->references) {
+        return;
+    }
+
+    const DetectReference *kv = pa->s->references;
+    jb_open_array(jb, "references");
+    while (kv) {
+        /* Note that the key and reference sizes have been bound
+         * checked during parsing
+         */
+        const size_t size_needed = kv->key_len + kv->reference_len + 1;
+        char kv_store[size_needed];
+        snprintf(kv_store, size_needed, "%s%s", kv->key, kv->reference);
+        jb_append_string(jb, kv_store);
+        kv = kv->next;
+    }
+    jb_close(jb);
+}
+
+static void AlertJsonMetadata(const PacketAlert *pa, JsonBuilder *js)
 {
     if (pa->s->metadata && pa->s->metadata->json_str) {
         jb_set_formatted(js, pa->s->metadata->json_str);
     }
 }
 
-void AlertJsonHeader(void *ctx, const Packet *p, const PacketAlert *pa, JsonBuilder *js,
-        uint16_t flags, JsonAddrInfo *addr, char *xff_buffer)
+void AlertJsonHeader(const Packet *p, const PacketAlert *pa, JsonBuilder *js, uint16_t flags,
+        JsonAddrInfo *addr, char *xff_buffer)
 {
-    AlertJsonOutputCtx *json_output_ctx = (AlertJsonOutputCtx *)ctx;
     const char *action = "allowed";
     /* use packet action if rate_filter modified the action */
     if (unlikely(pa->flags & PACKET_ALERT_RATE_FILTER_MODIFIED)) {
@@ -221,8 +242,12 @@ void AlertJsonHeader(void *ctx, const Packet *p, const PacketAlert *pa, JsonBuil
         AlertJsonSourceTarget(p, pa, js, addr);
     }
 
-    if ((json_output_ctx != NULL) && (flags & LOG_JSON_RULE_METADATA)) {
-        AlertJsonMetadata(json_output_ctx, pa, js);
+    if ((flags & LOG_JSON_REFERENCE)) {
+        AlertJsonReference(pa, js);
+    }
+
+    if (flags & LOG_JSON_RULE_METADATA) {
+        AlertJsonMetadata(pa, js);
     }
 
     if (flags & LOG_JSON_RULE) {
@@ -642,7 +667,7 @@ static int AlertJson(ThreadVars *tv, JsonAlertLogThread *aft, const Packet *p)
 
 
         /* alert */
-        AlertJsonHeader(json_output_ctx, p, pa, jb, json_output_ctx->flags, &addr, xff_buffer);
+        AlertJsonHeader(p, pa, jb, json_output_ctx->flags, &addr, xff_buffer);
 
         if (PacketIsTunnel(p)) {
             AlertJsonTunnel(p, jb);
@@ -774,7 +799,7 @@ static int AlertJsonDecoderEvent(ThreadVars *tv, JsonAlertLogThread *aft, const
         /* just the timestamp, no tuple */
         jb_set_string(jb, "timestamp", timebuf);
 
-        AlertJsonHeader(json_output_ctx, p, pa, jb, json_output_ctx->flags, NULL, NULL);
+        AlertJsonHeader(p, pa, jb, json_output_ctx->flags, NULL, NULL);
 
         OutputJsonBuilderBuffer(jb, aft->ctx);
         jb_free(jb);
@@ -902,6 +927,7 @@ static void JsonAlertLogSetupMetadata(AlertJsonOutputCtx *json_output_ctx,
                     SetFlag(rule_metadata, "raw", LOG_JSON_RULE, &flags);
                     SetFlag(rule_metadata, "metadata", LOG_JSON_RULE_METADATA,
                             &flags);
+                    SetFlag(rule_metadata, "reference", LOG_JSON_REFERENCE, &flags);
                 }
                 SetFlag(metadata, "flow", LOG_JSON_FLOW, &flags);
                 SetFlag(metadata, "app-layer", LOG_JSON_APP_LAYER, &flags);

src/output-json-alert.h

@@ -28,8 +28,8 @@
 #define SURICATA_OUTPUT_JSON_ALERT_H
 
 void JsonAlertLogRegister(void);
-void AlertJsonHeader(void *ctx, const Packet *p, const PacketAlert *pa, JsonBuilder *js,
-        uint16_t flags, JsonAddrInfo *addr, char *xff_buffer);
+void AlertJsonHeader(const Packet *p, const PacketAlert *pa, JsonBuilder *js, uint16_t flags,
+        JsonAddrInfo *addr, char *xff_buffer);
 void EveAddVerdict(JsonBuilder *jb, const Packet *p);
 
 #endif /* SURICATA_OUTPUT_JSON_ALERT_H */

src/output-json-drop.c

@@ -178,15 +178,15 @@ static int DropLogJSON (JsonDropLogThread *aft, const Packet *p)
             if ((pa->action & (ACTION_REJECT|ACTION_REJECT_DST|ACTION_REJECT_BOTH)) ||
                ((pa->action & ACTION_DROP) && EngineModeIsIPS()))
             {
-                AlertJsonHeader(NULL, p, pa, js, 0, &addr, NULL);
+                AlertJsonHeader(p, pa, js, 0, &addr, NULL);
                 logged = 1;
                 break;
             }
         }
         if (logged == 0) {
             if (p->alerts.drop.action != 0) {
                 const PacketAlert *pa = &p->alerts.drop;
-                AlertJsonHeader(NULL, p, pa, js, 0, &addr, NULL);
+                AlertJsonHeader(p, pa, js, 0, &addr, NULL);
             }
         }
     }