rapid7_insightvm: canonicalize host.name to lower case and map subdomain to host.hostname (elastic#9665)

efd6 · web-flow · commit dba2901594da · 2024-04-25T00:51:30.000Z
diff --git a/packages/rapid7_insightvm/changelog.yml b/packages/rapid7_insightvm/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.10.0"
+  changes:
+    - description: Canonicalize `host.name` to lower case and map subdomain to `host.hostname`.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/9665
 - version: "1.9.0"
   changes:
     - description: Set sensitive values as secret.
diff --git a/packages/rapid7_insightvm/data_stream/asset/_dev/test/pipeline/test-asset.log b/packages/rapid7_insightvm/data_stream/asset/_dev/test/pipeline/test-asset.log
@@ -1,3 +1,3 @@
 {"assessed_for_policies":false,"assessed_for_vulnerabilities":true,"critical_vulnerabilities":0,"exploits":0,"id":"452534235-25a7-40a3-9321-28ce0b5cc90e-default-asset-199","ip":"10.1.0.128","last_assessed_for_vulnerabilities":"2020-03-20T19:19:42.611Z","last_scan_end":"2020-03-20T19:19:42.611Z","last_scan_start":"2020-03-20T19:18:13.611Z","malware_kits":0,"moderate_vulnerabilities":2,"os_architecture":"x86_64","os_description":"CentOS Linux 2.6.18","os_family":"Linux","os_name":"Linux","os_system_name":"CentOS Linux","os_type":"General","os_vendor":"CentOS","os_version":"2.6.18","risk_score":0,"severe_vulnerabilities":0,"tags":[{"name":"lab","type":"SITE"}],"total_vulnerabilities":2,"new":[],"remediated":[]}
-{"assessed_for_policies":false,"assessed_for_vulnerabilities":true,"critical_vulnerabilities":1,"exploits":9,"host_name":"host.domain.com","id":"452534235-25a7-40a3-9321-28ce0b5cc90e-default-asset-198","ip":"10.4.24.164","last_scan_end":"2020-03-20T19:12:39.766Z","last_scan_start":"2020-03-20T19:05:06.766Z","malware_kits":0,"moderate_vulnerabilities":11,"os_architecture":"","os_description":"Ubuntu Linux 12.04","os_family":"Linux","os_name":"Linux","os_system_name":"Ubuntu Linux","os_type":"","os_vendor":"Ubuntu","os_version":"12.04","risk_score":12251.76171875,"severe_vulnerabilities":16,"tags":[{"name":"all_assets2","type":"CUSTOM"},{"name":"all_assets","type":"CUSTOM"},{"name":"Linux","type":"CUSTOM"},{"name":"docker hosts","type":"SITE"},{"name":"lab","type":"SITE"}],"total_vulnerabilities":28,"new":[],"remediated":[],"unique_identifiers":{"id":"4421d73dfe04f594df731e6bcd8156a","source":"R7 Agent"}}
+{"assessed_for_policies":false,"assessed_for_vulnerabilities":true,"critical_vulnerabilities":1,"exploits":9,"host_name":"HOST.domain.com","id":"452534235-25a7-40a3-9321-28ce0b5cc90e-default-asset-198","ip":"10.4.24.164","last_scan_end":"2020-03-20T19:12:39.766Z","last_scan_start":"2020-03-20T19:05:06.766Z","malware_kits":0,"moderate_vulnerabilities":11,"os_architecture":"","os_description":"Ubuntu Linux 12.04","os_family":"Linux","os_name":"Linux","os_system_name":"Ubuntu Linux","os_type":"","os_vendor":"Ubuntu","os_version":"12.04","risk_score":12251.76171875,"severe_vulnerabilities":16,"tags":[{"name":"all_assets2","type":"CUSTOM"},{"name":"all_assets","type":"CUSTOM"},{"name":"Linux","type":"CUSTOM"},{"name":"docker hosts","type":"SITE"},{"name":"lab","type":"SITE"}],"total_vulnerabilities":28,"new":[],"remediated":[],"unique_identifiers":{"id":"4421d73dfe04f594df731e6bcd8156a","source":"R7 Agent"}}
 {"data":[],"metadata":{"number":0,"size":0,"totalResources":2195,"totalPages":2195,"cursor":null},"links":[{"href":"https://us.api.insight.rapid7.com:443/vm/v4/integration/assets?page=0&size=2","rel":"first"},{"href":"https://us.api.insight.rapid7.com:443/vm/v4/integration/assets?page=0&size=2","rel":"self"},{"href":"https://us.api.insight.rapid7.com:443/vm/v4/integration/assets?page=1097&size=2","rel":"last"}]}
diff --git a/packages/rapid7_insightvm/data_stream/asset/_dev/test/pipeline/test-asset.log-expected.json b/packages/rapid7_insightvm/data_stream/asset/_dev/test/pipeline/test-asset.log-expected.json
@@ -85,12 +85,13 @@
                     "host"
                 ],
                 "kind": "state",
-                "original": "{\"assessed_for_policies\":false,\"assessed_for_vulnerabilities\":true,\"critical_vulnerabilities\":1,\"exploits\":9,\"host_name\":\"host.domain.com\",\"id\":\"452534235-25a7-40a3-9321-28ce0b5cc90e-default-asset-198\",\"ip\":\"10.4.24.164\",\"last_scan_end\":\"2020-03-20T19:12:39.766Z\",\"last_scan_start\":\"2020-03-20T19:05:06.766Z\",\"malware_kits\":0,\"moderate_vulnerabilities\":11,\"os_architecture\":\"\",\"os_description\":\"Ubuntu Linux 12.04\",\"os_family\":\"Linux\",\"os_name\":\"Linux\",\"os_system_name\":\"Ubuntu Linux\",\"os_type\":\"\",\"os_vendor\":\"Ubuntu\",\"os_version\":\"12.04\",\"risk_score\":12251.76171875,\"severe_vulnerabilities\":16,\"tags\":[{\"name\":\"all_assets2\",\"type\":\"CUSTOM\"},{\"name\":\"all_assets\",\"type\":\"CUSTOM\"},{\"name\":\"Linux\",\"type\":\"CUSTOM\"},{\"name\":\"docker hosts\",\"type\":\"SITE\"},{\"name\":\"lab\",\"type\":\"SITE\"}],\"total_vulnerabilities\":28,\"new\":[],\"remediated\":[],\"unique_identifiers\":{\"id\":\"4421d73dfe04f594df731e6bcd8156a\",\"source\":\"R7 Agent\"}}",
+                "original": "{\"assessed_for_policies\":false,\"assessed_for_vulnerabilities\":true,\"critical_vulnerabilities\":1,\"exploits\":9,\"host_name\":\"HOST.domain.com\",\"id\":\"452534235-25a7-40a3-9321-28ce0b5cc90e-default-asset-198\",\"ip\":\"10.4.24.164\",\"last_scan_end\":\"2020-03-20T19:12:39.766Z\",\"last_scan_start\":\"2020-03-20T19:05:06.766Z\",\"malware_kits\":0,\"moderate_vulnerabilities\":11,\"os_architecture\":\"\",\"os_description\":\"Ubuntu Linux 12.04\",\"os_family\":\"Linux\",\"os_name\":\"Linux\",\"os_system_name\":\"Ubuntu Linux\",\"os_type\":\"\",\"os_vendor\":\"Ubuntu\",\"os_version\":\"12.04\",\"risk_score\":12251.76171875,\"severe_vulnerabilities\":16,\"tags\":[{\"name\":\"all_assets2\",\"type\":\"CUSTOM\"},{\"name\":\"all_assets\",\"type\":\"CUSTOM\"},{\"name\":\"Linux\",\"type\":\"CUSTOM\"},{\"name\":\"docker hosts\",\"type\":\"SITE\"},{\"name\":\"lab\",\"type\":\"SITE\"}],\"total_vulnerabilities\":28,\"new\":[],\"remediated\":[],\"unique_identifiers\":{\"id\":\"4421d73dfe04f594df731e6bcd8156a\",\"source\":\"R7 Agent\"}}",
                 "type": [
                     "info"
                 ]
             },
             "host": {
+                "hostname": "host",
                 "id": "452534235-25a7-40a3-9321-28ce0b5cc90e-default-asset-198",
                 "ip": [
                     "10.4.24.164"
@@ -113,7 +114,7 @@
                         "assessed_for_vulnerabilities": true,
                         "critical_vulnerabilities": 1,
                         "exploits": 9,
-                        "host_name": "host.domain.com",
+                        "host_name": "HOST.domain.com",
                         "id": "452534235-25a7-40a3-9321-28ce0b5cc90e-default-asset-198",
                         "ip": "10.4.24.164",
                         "last_scan_end": "2020-03-20T19:12:39.766Z",
@@ -162,7 +163,8 @@
             },
             "related": {
                 "hosts": [
-                    "host.domain.com"
+                    "host.domain.com",
+                    "host"
                 ],
                 "ip": [
                     "10.4.24.164"
diff --git a/packages/rapid7_insightvm/data_stream/asset/elasticsearch/ingest_pipeline/default.yml b/packages/rapid7_insightvm/data_stream/asset/elasticsearch/ingest_pipeline/default.yml
@@ -115,10 +115,28 @@ processors:
       field: host.name
       copy_from: rapid7.insightvm.asset.host_name
       ignore_empty_value: true
+  - lowercase:
+      field: host.name
+      ignore_missing: true
+  - append:
+      field: related.hosts
+      value: '{{{host.name}}}'
+      if: ctx.host?.name != null
+      allow_duplicates: false
+  - script:
+      lang: painless
+      if: ctx.host?.name != null
+      source: |
+        int idx = ctx.host.name.indexOf(".");
+        if (idx == -1) {
+          ctx.host.hostname = ctx.host.name;
+        } else {
+          ctx.host.hostname = ctx.host.name.substring(0, idx);
+        }
   - append:
       field: related.hosts
-      value: '{{{rapid7.insightvm.asset.host_name}}}'
-      if: ctx.rapid7?.insightvm?.asset?.host_name != null
+      value: '{{{host.hostname}}}'
+      if: ctx.host?.hostname != null
       allow_duplicates: false
   - rename:
       field: json.id
diff --git a/packages/rapid7_insightvm/manifest.yml b/packages/rapid7_insightvm/manifest.yml
@@ -1,7 +1,7 @@
 format_version: "3.0.2"
 name: rapid7_insightvm
 title: Rapid7 InsightVM
-version: "1.9.0"
+version: "1.10.0"
 source:
   license: "Elastic-2.0"
 description: Collect logs from Rapid7 InsightVM with Elastic Agent.
