Skip to content

Commit eda4138

Browse files
efd6efd6
authored
crowdstrike: fix handling of network direction (#12508)
Assume that network direction that is not inbound can be validly semantically represented as outbound using the ECS fields available. This is probably not true; documented values of the ConnectionDirection are 0 - outbound, 1 - inbound, 2 - neither, and 3 - both[1]. Adhering strictly to inbound/outbound makes it impossible to map the data to ECS since neither and both would only be expressible as unknown. [1]https://docs.panther.com/data-onboarding/supported-logs/crowdstrike/falcon-data-replicator#crowdstrike.networkconnect
1 parent 4060f32 commit eda4138

File tree

8 files changed

+256
-112
lines changed

8 files changed

+256
-112
lines changed

‎packages/crowdstrike/changelog.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,12 @@
11
# newer versions go on top
2+
- version: "1.49.1"
3+
changes:
4+
- description: Fix network direction handling for FDR data stream.
5+
type: bugfix
6+
link: https://github.com/elastic/integrations/pull/12508
7+
- description: Handle invalid IP addresses robustly.
8+
type: bugfix
9+
link: https://github.com/elastic/integrations/pull/12508
210
- version: "1.49.0"
311
changes:
412
- description: Add "preserve_original_event" tag to documents with `event.kind` manually set to "pipeline_error".

‎packages/crowdstrike/data_stream/fdr/_dev/test/pipeline/test-data.log-expected.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -316,4 +316,4 @@
316316
}
317317
}
318318
]
319-
}
319+
}

‎packages/crowdstrike/data_stream/fdr/_dev/test/pipeline/test-fdr.log-expected.json‎

Lines changed: 37 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -593,21 +593,9 @@
593593
"name": "NetworkReceiveAcceptIP4LinV5"
594594
},
595595
"destination": {
596-
"address": "67.43.156.14",
597-
"as": {
598-
"number": 35908
599-
},
600-
"geo": {
601-
"continent_name": "Asia",
602-
"country_iso_code": "BT",
603-
"country_name": "Bhutan",
604-
"location": {
605-
"lat": 27.5,
606-
"lon": 90.5
607-
}
608-
},
609-
"ip": "67.43.156.14",
610-
"port": 53
596+
"address": "0.0.0.0",
597+
"ip": "0.0.0.0",
598+
"port": 39920
611599
},
612600
"ecs": {
613601
"version": "8.11.0"
@@ -673,9 +661,21 @@
673661
]
674662
},
675663
"source": {
676-
"address": "0.0.0.0",
677-
"ip": "0.0.0.0",
678-
"port": 39920
664+
"address": "67.43.156.14",
665+
"as": {
666+
"number": 35908
667+
},
668+
"geo": {
669+
"continent_name": "Asia",
670+
"country_iso_code": "BT",
671+
"country_name": "Bhutan",
672+
"location": {
673+
"lat": 27.5,
674+
"lon": 90.5
675+
}
676+
},
677+
"ip": "67.43.156.14",
678+
"port": 53
679679
},
680680
"tags": [
681681
"preserve_original_event"
@@ -7698,21 +7698,9 @@
76987698
"name": "NetworkConnectIP4MacV5"
76997699
},
77007700
"destination": {
7701-
"address": "67.43.156.14",
7702-
"as": {
7703-
"number": 35908
7704-
},
7705-
"geo": {
7706-
"continent_name": "Asia",
7707-
"country_iso_code": "BT",
7708-
"country_name": "Bhutan",
7709-
"location": {
7710-
"lat": 27.5,
7711-
"lon": 90.5
7712-
}
7713-
},
7714-
"ip": "67.43.156.14",
7715-
"port": 443
7701+
"address": "0.0.0.0",
7702+
"ip": "0.0.0.0",
7703+
"port": 0
77167704
},
77177705
"ecs": {
77187706
"version": "8.11.0"
@@ -7777,9 +7765,21 @@
77777765
]
77787766
},
77797767
"source": {
7780-
"address": "0.0.0.0",
7781-
"ip": "0.0.0.0",
7782-
"port": 0
7768+
"address": "67.43.156.14",
7769+
"as": {
7770+
"number": 35908
7771+
},
7772+
"geo": {
7773+
"continent_name": "Asia",
7774+
"country_iso_code": "BT",
7775+
"country_name": "Bhutan",
7776+
"location": {
7777+
"lat": 27.5,
7778+
"lon": 90.5
7779+
}
7780+
},
7781+
"ip": "67.43.156.14",
7782+
"port": 443
77837783
},
77847784
"tags": [
77857785
"preserve_original_event"
@@ -12092,4 +12092,4 @@
1209212092
}
1209312093
}
1209412094
]
12095-
}
12095+
}

‎packages/crowdstrike/data_stream/fdr/_dev/test/pipeline/test-fdrv2-notmanaged.log-expected.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,4 +68,4 @@
6868
}
6969
}
7070
]
71-
}
71+
}

‎packages/crowdstrike/data_stream/fdr/elasticsearch/ingest_pipeline/default.yml‎

Lines changed: 28 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -1925,16 +1925,21 @@ processors:
19251925

19261926
## Networking fields.
19271927
- set:
1928-
field: source.ip
1929-
if: ctx.source?.ip == null && ctx.crowdstrike?.CurrentLocalIP != null
1930-
value: "{{{crowdstrike.CurrentLocalIP}}}"
1928+
field: network.direction
1929+
value: outbound
1930+
if: ctx.crowdstrike?.ConnectionDirection == "0"
19311931
- set:
1932-
field: source.ip
1933-
if: ctx.source?.ip == null && ctx.crowdstrike.LocalIP != null
1934-
value: "{{{crowdstrike.LocalIP}}}"
1932+
field: network.direction
1933+
value: inbound
1934+
if: ctx.crowdstrike?.ConnectionDirection == "1"
1935+
- set:
1936+
field: network.direction
1937+
value: unknown
1938+
if: ctx.network?.direction == null && ctx.crowdstrike?.ConnectionDirection != null && ctx.crowdstrike.ConnectionDirection != ""
1939+
19351940
- split:
19361941
field: crowdstrike.LocalAddressIP4
1937-
separator: "\\s+"
1942+
separator: '\s+'
19381943
if: ctx.crowdstrike?.LocalAddressIP4 != null
19391944
- convert:
19401945
tag: convert_LocalAddressIP4_ip
@@ -1945,21 +1950,9 @@ processors:
19451950
- append:
19461951
field: error.message
19471952
value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
1948-
- set:
1949-
field: source.ip
1950-
if: ctx.source?.ip == null && ctx.crowdstrike?.LocalAddressIP4 instanceof List && ctx.crowdstrike.LocalAddressIP4.length > 0
1951-
value: "{{{crowdstrike.LocalAddressIP4.0}}}"
1952-
# - foreach:
1953-
# if: ctx.crowdstrike?.LocalAddressIP4 instanceof List && ctx.crowdstrike.LocalAddressIP4.length > 0
1954-
# field: crowdstrike.LocalAddressIP4
1955-
# processor:
1956-
# append:
1957-
# field: source.ip
1958-
# value: '{{{_ingest._value}}}'
1959-
# allow_duplicates: false
19601953
- split:
19611954
field: crowdstrike.LocalAddressIP6
1962-
separator: "\\s+"
1955+
separator: '\s+'
19631956
if: ctx.crowdstrike?.LocalAddressIP6 != null
19641957
- convert:
19651958
tag: convert_LocalAddressIP6_ip
@@ -1970,56 +1963,31 @@ processors:
19701963
- append:
19711964
field: error.message
19721965
value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
1973-
- set:
1974-
field: source.ip
1975-
if: ctx.source?.ip == null && ctx.crowdstrike?.LocalAddressIP6 instanceof List && ctx.crowdstrike.LocalAddressIP6.length > 0
1976-
value: "{{{crowdstrike.LocalAddressIP6.0}}}"
1977-
# - foreach:
1978-
# if: ctx.crowdstrike?.LocalAddressIP6 != null && ctx.crowdstrike.LocalAddressIP6 instanceof List && ctx.crowdstrike.LocalAddressIP6.length > 0
1979-
# field: crowdstrike.LocalAddressIP6
1980-
# processor:
1981-
# append:
1982-
# field: source.ip
1983-
# value: '{{{_ingest._value}}}'
1984-
# allow_duplicates: false
1985-
- set:
1986-
field: source.address
1987-
copy_from: source.ip
1988-
ignore_empty_value: true
1989-
- rename:
1990-
field: crowdstrike.LocalPort
1991-
target_field: source.port
1992-
ignore_missing: true
19931966
- convert:
19941967
tag: convert_RemoteAddressIP4_ip
19951968
field: crowdstrike.RemoteAddressIP4
19961969
type: ip
19971970
ignore_missing: true
1998-
- rename:
1999-
field: crowdstrike.RemoteAddressIP4
2000-
target_field: destination.ip
2001-
ignore_missing: true
2002-
- set:
2003-
field: destination.address
2004-
copy_from: destination.ip
2005-
ignore_empty_value: true
20061971
- convert:
20071972
tag: convert_RemoteAddressIP6_ip
20081973
field: crowdstrike.RemoteAddressIP6
20091974
type: ip
20101975
ignore_missing: true
2011-
- rename:
2012-
field: crowdstrike.RemoteAddressIP6
2013-
target_field: destination.ip
2014-
ignore_missing: true
2015-
- set:
2016-
field: destination.address
2017-
copy_from: destination.ip
2018-
ignore_empty_value: true
2019-
- rename:
2020-
field: crowdstrike.RemotePort
2021-
target_field: destination.port
2022-
ignore_missing: true
1976+
1977+
- pipeline:
1978+
tag: pipeline_outbound_network
1979+
# The condition is all non-inbound, but the pipeline operates assuming the traffic is outbound.
1980+
# In cases where there is no information we make this assumption rather than dropping the data
1981+
# on the floor.
1982+
if: ctx.network?.direction != 'inbound'
1983+
name: '{{ IngestPipeline "outbound_network" }}'
1984+
ignore_missing_pipeline: true
1985+
- pipeline:
1986+
tag: pipeline_inbound_network
1987+
if: ctx.network?.direction == 'inbound'
1988+
name: '{{ IngestPipeline "inbound_network" }}'
1989+
ignore_missing_pipeline: true
1990+
20231991
- rename:
20241992
field: crowdstrike.Protocol
20251993
target_field: network.iana_number
@@ -2054,18 +2022,6 @@ processors:
20542022
} else if (iana_number == '132') {
20552023
ctx.network.transport = 'sctp';
20562024
}
2057-
- set:
2058-
field: network.direction
2059-
value: outbound
2060-
if: ctx.crowdstrike?.ConnectionDirection == "0"
2061-
- set:
2062-
field: network.direction
2063-
value: inbound
2064-
if: ctx.crowdstrike?.ConnectionDirection == "1"
2065-
- set:
2066-
field: network.direction
2067-
value: unknown
2068-
if: ctx.network?.direction == null && ctx.crowdstrike?.ConnectionDirection != null && ctx.crowdstrike.ConnectionDirection != ""
20692025
- community_id:
20702026
ignore_missing: true
20712027
ignore_failure: true

‎packages/crowdstrike/data_stream/fdr/elasticsearch/ingest_pipeline/inbound_network.yml‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
---
2+
description: Pipeline for processing inbound network details
3+
processors:
4+
- set:
5+
field: destination.ip
6+
if: ctx.destination?.ip == null && ctx.crowdstrike?.CurrentLocalIP != null
7+
value: '{{{crowdstrike.CurrentLocalIP}}}'
8+
- set:
9+
field: destination.ip
10+
if: ctx.destination?.ip == null && ctx.crowdstrike.LocalIP != null
11+
value: '{{{crowdstrike.LocalIP}}}'
12+
- set:
13+
field: destination.ip
14+
if: ctx.destination?.ip == null && ctx.crowdstrike?.LocalAddressIP4 instanceof List && ctx.crowdstrike.LocalAddressIP4.length > 0
15+
value: '{{{crowdstrike.LocalAddressIP4.0}}}'
16+
- set:
17+
field: destination.ip
18+
if: ctx.destination?.ip == null && ctx.crowdstrike?.LocalAddressIP6 instanceof List && ctx.crowdstrike.LocalAddressIP6.length > 0
19+
value: '{{{crowdstrike.LocalAddressIP6.0}}}'
20+
- convert:
21+
tag: convert_destination_ip
22+
field: destination.ip
23+
type: ip
24+
ignore_missing: true
25+
on_failure:
26+
- remove:
27+
field: destination.ip
28+
ignore_missing: true
29+
- append:
30+
field: error.message
31+
value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
32+
- set:
33+
field: destination.address
34+
copy_from: destination.ip
35+
ignore_empty_value: true
36+
- rename:
37+
field: crowdstrike.LocalPort
38+
target_field: destination.port
39+
ignore_missing: true
40+
41+
- rename:
42+
field: crowdstrike.MAC
43+
target_field: destination.mac
44+
ignore_missing: true
45+
- rename:
46+
if: ctx.destination?.mac == null
47+
field: crowdstrike.PhysicalAddress
48+
target_field: destination.mac
49+
ignore_missing: true
50+
51+
- convert:
52+
tag: convert_RemoteAddressIP4_ip
53+
field: crowdstrike.RemoteAddressIP4
54+
type: ip
55+
ignore_missing: true
56+
on_failure:
57+
- remove:
58+
field: crowdstrike.RemoteAddressIP4
59+
ignore_missing: true
60+
- append:
61+
field: error.message
62+
value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
63+
- rename:
64+
field: crowdstrike.RemoteAddressIP4
65+
target_field: source.ip
66+
ignore_missing: true
67+
- convert:
68+
tag: convert_RemoteAddressIP6_ip
69+
field: crowdstrike.RemoteAddressIP6
70+
type: ip
71+
ignore_missing: true
72+
on_failure:
73+
- remove:
74+
field: crowdstrike.RemoteAddressIP6
75+
ignore_missing: true
76+
- append:
77+
field: error.message
78+
value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
79+
- rename:
80+
field: crowdstrike.RemoteAddressIP6
81+
target_field: source.ip
82+
ignore_missing: true
83+
- set:
84+
field: source.address
85+
copy_from: source.ip
86+
ignore_empty_value: true
87+
- rename:
88+
field: crowdstrike.RemotePort
89+
target_field: source.port
90+
ignore_missing: true

0 commit comments

Comments
 (0)