diff --git a/Makefile.am b/Makefile.am index 316b93c0..389485db 100644 --- a/Makefile.am +++ b/Makefile.am @@ -80,8 +80,6 @@ ipfixprobe_output_src+=\ fields.h endif - - ipfixprobe_process_src=\ process/http.cpp \ process/http.hpp \ @@ -130,6 +128,7 @@ ipfixprobe_process_src+=\ process/quic.hpp \ process/quic.cpp endif + if WITH_FLEXPROBE ipfixprobe_process_src+=\ process/flexprobe-data.h \ @@ -141,6 +140,12 @@ ipfixprobe_process_src+=\ process/flexprobe-encryption-processing.h endif +if WITH_OSQUERY +ipfixprobe_input_src+=\ + process/osquery.cpp\ + process/osquery.hpp +endif + if WITH_DPDK ipfixprobe_input_src+=\ input/dpdk.cpp \ diff --git a/README.md b/README.md index c7d52058..4593214d 100644 --- a/README.md +++ b/README.md @@ -414,6 +414,23 @@ Note: the following fields are UniRec arrays. ipfixprobe -p pstats:includezeros -r sample.pcap -i "f:output.trapcap" ``` +### OSQUERY +List of unirec fields exported together with basic flow fields on interface by OSQUERY plugin. + +| UniRec field | Type | Description | +|:--------------------------:|:--------:|:---------------------------------------------------:| +| PROGRAM_NAME | string | The name of the program that handles the connection | +| USERNAME | string | The name of the user who starts the process | +| OS_NAME | string | Distribution or product name | +| OS_MAJOR | uint16 | Major release version | +| OS_MINOR | uint16 | Minor release version | +| OS_BUILD | string | Optional build-specific or variant string | +| OS_PLATFORM | string | OS Platform or ID | +| OS_PLATFORM_LIKE | string | Closely related platforms | +| OS_ARCH | string | OS Architecture | +| KERNEL_VERSION | string | Kernel version | +| SYSTEM_HOSTNAME | string | Network hostname including domain | + ### SSDP List of unirec fields exported together with basic flow fields on interface by SSDP plugin. diff --git a/configure.ac b/configure.ac index dc79461b..82bf4d91 100644 --- a/configure.ac +++ b/configure.ac @@ -293,6 +293,25 @@ RPM_REQUIRES+=" libtrap" RPM_BUILDREQ+=" libtrap-devel unirec" fi +AC_ARG_WITH([osquery], + AC_HELP_STRING([--with-osquery],[Compile with osquery framework (osquery.io).]), + [ + if test "$withval" = "yes"; then + withosquery="yes" + AC_CHECK_PROG(OSQUERY, osqueryi, yes) + AS_IF([test x${OSQUERY} != xyes], [AC_MSG_ERROR([Please install osquery before configuring.])]) + else + withosquery="no" + fi + ], [withosquery="no"] +) + +AM_CONDITIONAL(WITH_OSQUERY, test x${withosquery} = xyes) + +if [[ -z "$WITH_OSQUERY_TRUE" ]]; then + AC_DEFINE([WITH_OSQUERY], [1], [Define to 1 if the osquery is available]) +fi + AC_ARG_WITH([dpdk], AS_HELP_STRING([--with-dpdk],[Compile ipfixprobe with DPDK interface support.]), diff --git a/include/ipfixprobe/ipfix-elements.hpp b/include/ipfixprobe/ipfix-elements.hpp index 51c97f85..cbdf362c 100644 --- a/include/ipfixprobe/ipfix-elements.hpp +++ b/include/ipfixprobe/ipfix-elements.hpp @@ -231,7 +231,6 @@ namespace ipxp { #define STATS_PCKT_TCPFLGS(F) F(0, 291, -1, nullptr) // BASIC LIST -- FIELD IS e8057id1015 (uint8*) #define STATS_PCKT_DIRECTIONS(F) F(0, 291, -1, nullptr) // BASIC LIST -- FIELD IS e8057id1016 (int8*) - #define SBI_BRST_PACKETS(F) F(0, 291, -1, nullptr) // BASIC LIST -- FIELD IS e8057id1050 (uint16*) #define SBI_BRST_BYTES(F) F(0, 291, -1, nullptr) // BASIC LIST -- FIELD IS e8057id1051 (uint16*) #define SBI_BRST_TIME_START(F) F(0, 291, -1, nullptr) // BASIC LIST -- FIELD IS e8057id1052 (time*) @@ -252,9 +251,21 @@ namespace ipxp { #define QUIC_SNI(F) F(8057, 890, -1, nullptr) +#define OSQUERY_PROGRAM_NAME(F) F(8057, 852, -1, nullptr) +#define OSQUERY_USERNAME(F) F(8057, 853, -1, nullptr) +#define OSQUERY_OS_NAME(F) F(8057, 854, -1, nullptr) +#define OSQUERY_OS_MAJOR(F) F(8057, 855, 2, nullptr) +#define OSQUERY_OS_MINOR(F) F(8057, 856, 2, nullptr) +#define OSQUERY_OS_BUILD(F) F(8057, 857, -1, nullptr) +#define OSQUERY_OS_PLATFORM(F) F(8057, 858, -1, nullptr) +#define OSQUERY_OS_PLATFORM_LIKE(F) F(8057, 859, -1, nullptr) +#define OSQUERY_OS_ARCH(F) F(8057, 860, -1, nullptr) +#define OSQUERY_KERNEL_VERSION(F) F(8057, 861, -1, nullptr) +#define OSQUERY_SYSTEM_HOSTNAME(F) F(8057, 862, -1, nullptr) + #ifdef WITH_FLEXPROBE -#define FX_FRAME_SIGNATURE(F) F(5715, 1010, 18, NULL) -#define FX_TCP_TRACKING(F) F(5715, 1020, 1, NULL) +#define FX_FRAME_SIGNATURE(F) F(5715, 1010, 18, nullptr) +#define FX_TCP_TRACKING(F) F(5715, 1020, 1, nullptr) #endif /** @@ -467,6 +478,19 @@ namespace ipxp { #define IPFIX_QUIC_TEMPLATE(F) \ F(QUIC_SNI) +#define IPFIX_OSQUERY_TEMPLATE(F) \ + F(OSQUERY_PROGRAM_NAME) \ + F(OSQUERY_USERNAME) \ + F(OSQUERY_OS_NAME) \ + F(OSQUERY_OS_MAJOR) \ + F(OSQUERY_OS_MINOR) \ + F(OSQUERY_OS_BUILD) \ + F(OSQUERY_OS_PLATFORM) \ + F(OSQUERY_OS_PLATFORM_LIKE) \ + F(OSQUERY_OS_ARCH) \ + F(OSQUERY_KERNEL_VERSION) \ + F(OSQUERY_SYSTEM_HOSTNAME) + #ifdef WITH_FLEXPROBE #define IPFIX_FLEXPROBE_DATA_TEMPLATE(F) F(FX_FRAME_SIGNATURE) #define IPFIX_FLEXPROBE_TCP_TEMPLATE(F) F(FX_TCP_TRACKING) @@ -505,6 +529,7 @@ namespace ipxp { IPFIX_PHISTS_TEMPLATE(F) \ IPFIX_WG_TEMPLATE(F) \ IPFIX_QUIC_TEMPLATE(F) \ + IPFIX_OSQUERY_TEMPLATE(F) \ IPFIX_FLEXPROBE_DATA_TEMPLATE(F) \ IPFIX_FLEXPROBE_TCP_TEMPLATE(F) \ IPFIX_FLEXPROBE_ENCR_TEMPLATE(F) diff --git a/ipfix-elements.h b/ipfix-elements.h deleted file mode 100644 index a0083215..00000000 --- a/ipfix-elements.h +++ /dev/null @@ -1,486 +0,0 @@ -/** - * \file ipfix-elements.h - * \brief List of IPFIX elements and templates - * \author Tomas Cejka - * \date 2017 - * \date 2020 - */ -/* - * Copyright (C) 2020 CESNET - * - * LICENSE TERMS - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * 3. Neither the name of the Company nor the names of its contributors - * may be used to endorse or promote products derived from this - * software without specific prior written permission. - * - * ALTERNATIVELY, provided that this notice is retained in full, this - * product may be distributed under the terms of the GNU General Public - * License (GPL) version 2 or later, in which case the provisions - * of the GPL apply INSTEAD OF those given above. - * - * This software is provided ``as is'', and any express or implied - * warranties, including, but not limited to, the implied warranties of - * merchantability and fitness for a particular purpose are disclaimed. - * In no event shall the company or contributors be liable for any - * direct, indirect, incidental, special, exemplary, or consequential - * damages (including, but not limited to, procurement of substitute - * goods or services; loss of use, data, or profits; or business - * interruption) however caused and on any theory of liability, whether - * in contract, strict liability, or tort (including negligence or - * otherwise) arising in any way out of the use of this software, even - * if advised of the possibility of such damage. - * - */ - -#ifndef IPFIX_ELEMENTS_H -#define IPFIX_ELEMENTS_H - - -/** - * Each IPFIX element is defined as a C-preprocessor macro expecting - * one argument - macro function that is used to pass 4 arguments (info about an element). - * - * The IPFIX element has 4 "attributes" in the following order: - * 1. Enterprise number, - * 2. Element ID, - * 3. Data type length (in bytes), - * 4. Source memory pointer (to copy value from) - */ - - -/** - * Difference between NTP and UNIX epoch in number of seconds. - */ -#define EPOCH_DIFF 2208988800ULL - -/** - * Conversion from microseconds to NTP fraction (resolution 1/(2^32)s, ~233 picoseconds). - * Division by 1000000 would lead to wrong value when converting fraction back to microseconds, so 999999 is used. - */ -#define NTP_USEC_TO_FRAC(usec) (uint32_t)(((uint64_t) usec << 32) / 999999) - -/** - * Create 64 bit NTP timestamp which consist of 32 bit seconds part and 32 bit fraction part. - */ -#define MK_NTP_TS(ts) (((uint64_t) (ts.tv_sec + EPOCH_DIFF) << 32) | (uint64_t) NTP_USEC_TO_FRAC(ts.tv_usec)) - -/** - * Convert FIELD to its "attributes", i.e. BYTES(FIELD) used in the source code produces - * 0, 1, 8, &flow.octet_total_length - * when it is substituted by C-preprocessor. - */ -#define FIELD(EN, ID, LEN, SRC) EN, ID, LEN, SRC - -/* The list of known IPFIX elements: */ -#define BYTES(F) F(0, 1, 8, &flow.src_octet_total_length) -#define BYTES_REV(F) F(29305, 1, 8, &flow.dst_octet_total_length) -#define PACKETS(F) F(0, 2, 8, (temp = (uint64_t) flow.src_pkt_total_cnt, &temp)) -#define PACKETS_REV(F) F(29305, 2, 8, (temp = (uint64_t) flow.dst_pkt_total_cnt, &temp)) -#define FLOW_START_USEC(F) F(0, 154, 8, (temp = MK_NTP_TS(flow.time_first), &temp)) -#define FLOW_END_USEC(F) F(0, 155, 8, (temp = MK_NTP_TS(flow.time_last), &temp)) -#define OBSERVATION_MSEC(F) F(0, 323, 8, NULL) -#define INPUT_INTERFACE(F) F(0, 10, 2, &this->dir_bit_field) -#define OUTPUT_INTERFACE(F) F(0, 14, 2, NULL) -#define FLOW_END_REASON(F) F(0, 136, 1, &flow.end_reason) - -#define ETHERTYPE(F) F(0, 256, 2, NULL) - -#define L2_SRC_MAC(F) F(0, 56, 6, flow.src_mac) -#define L2_DST_MAC(F) F(0, 80, 6, flow.dst_mac) - -#define L3_PROTO(F) F(0, 60, 1, &flow.ip_version) -#define L3_IPV4_ADDR_SRC(F) F(0, 8, 4, &flow.src_ip.v4) -#define L3_IPV4_ADDR_DST(F) F(0, 12, 4, &flow.dst_ip.v4) -#define L3_IPV4_TOS(F) F(0, 5, 1, NULL) -#define L3_IPV6_ADDR_SRC(F) F(0, 27, 16, &flow.src_ip.v6) -#define L3_IPV6_ADDR_DST(F) F(0, 28, 16, &flow.dst_ip.v6) -#define L3_IPV4_IDENTIFICATION(F) F(0, 54, 2, NULL) -#define L3_IPV4_FRAGMENT(F) F(0, 88, 2, NULL) -#define L3_IPV4_TTL(F) F(0, 192, 1, NULL) -#define L3_IPV6_TTL(F) F(0, 192, 1, NULL) -#define L3_TTL(F) F(0, 192, 1, NULL) -#define L3_TTL_REV(F) F(29305, 192, 1, NULL) -#define L3_FLAGS(F) F(0, 197, 1, NULL) -#define L3_FLAGS_REV(F) F(29305, 197, 1, NULL) - -#define L4_PROTO(F) F(0, 4, 1, &flow.ip_proto) -#define L4_TCP_FLAGS(F) F(0, 6, 1, &flow.src_tcp_control_bits) -#define L4_TCP_FLAGS_REV(F) F(29305, 6, 1, &flow.dst_tcp_control_bits) -#define L4_PORT_SRC(F) F(0, 7, 2, &flow.src_port) -#define L4_PORT_DST(F) F(0, 11, 2, &flow.dst_port) -#define L4_ICMP_TYPE_CODE(F) F(0, 32, 2, NULL) -#define L4_TCP_WIN(F) F(0, 186, 2, NULL) -#define L4_TCP_WIN_REV(F) F(29305, 186, 2, NULL) -#define L4_TCP_OPTIONS(F) F(0, 209, 8, NULL) -#define L4_TCP_OPTIONS_REV(F) F(29305, 209, 8, NULL) - - -#define L4_TCP_MSS(F) F(8057, 900, 4, NULL) -#define L4_TCP_MSS_REV(F) F(8057, 901, 4, NULL) -#define L4_TCP_SYN_SIZE(F) F(8057, 902, 2, NULL) - -#define HTTP_DOMAIN(F) F(39499, 1, -1, NULL) -#define HTTP_REFERER(F) F(39499, 3, -1, NULL) -#define HTTP_URI(F) F(39499, 2, -1, NULL) -#define HTTP_CONTENT_TYPE(F) F(39499, 10, -1, NULL) -#define HTTP_STATUS(F) F(39499, 12, 2, NULL) -#define HTTP_USERAGENT(F) F(39499, 20, -1, NULL) -#define HTTP_METHOD(F) F(8057, 200, -1, NULL) - -#define RTSP_METHOD(F) F(16982, 600, -1, NULL) -#define RTSP_USERAGENT(F) F(16982, 601, -1, NULL) -#define RTSP_URI(F) F(16982, 602, -1, NULL) -#define RTSP_STATUS(F) F(16982, 603, 2, NULL) -#define RTSP_CONTENT_TYPE(F) F(16982, 604, -1, NULL) -#define RTSP_SERVER(F) F(16982, 605, -1, NULL) - -#define DNS_RCODE(F) F(8057, 1, 1, NULL) -#define DNS_NAME(F) F(8057, 2, -1, NULL) -#define DNS_QTYPE(F) F(8057, 3, 2, NULL) -#define DNS_CLASS(F) F(8057, 4, 2, NULL) -#define DNS_RR_TTL(F) F(8057, 5, 4, NULL) -#define DNS_RLENGTH(F) F(8057, 6, 2, NULL) -#define DNS_RDATA(F) F(8057, 7, -1, NULL) -#define DNS_PSIZE(F) F(8057, 8, 2, NULL) -#define DNS_DO(F) F(8057, 9, 1, NULL) -#define DNS_ID(F) F(8057, 10, 2, NULL) -#define DNS_ATYPE(F) F(8057, 11, 2, NULL) -#define DNS_ANSWERS(F) F(8057, 14, 2, NULL) - -#define SIP_MSG_TYPE(F) F(8057, 100, 2, NULL) -#define SIP_STATUS_CODE(F) F(8057, 101, 2, NULL) -#define SIP_CALL_ID(F) F(8057, 102, -1, NULL) -#define SIP_CALLING_PARTY(F) F(8057, 103, -1, NULL) -#define SIP_CALLED_PARTY(F) F(8057, 104, -1, NULL) -#define SIP_VIA(F) F(8057, 105, -1, NULL) -#define SIP_USER_AGENT(F) F(8057, 106, -1, NULL) -#define SIP_REQUEST_URI(F) F(8057, 107, -1, NULL) -#define SIP_CSEQ(F) F(8057, 108, -1, NULL) - -#define NTP_LEAP(F) F(8057, 18, 1, NULL) -#define NTP_VERSION(F) F(8057, 19, 1, NULL) -#define NTP_MODE(F) F(8057, 20, 1, NULL) -#define NTP_STRATUM(F) F(8057, 21, 1, NULL) -#define NTP_POLL(F) F(8057, 22, 1, NULL) -#define NTP_PRECISION(F) F(8057, 23, 1, NULL) -#define NTP_DELAY(F) F(8057, 24, 4, NULL) -#define NTP_DISPERSION(F) F(8057, 25, 4, NULL) -#define NTP_REF_ID(F) F(8057, 26, -1, NULL) -#define NTP_REF(F) F(8057, 27, -1, NULL) -#define NTP_ORIG(F) F(8057, 28, -1, NULL) -#define NTP_RECV(F) F(8057, 29, -1, NULL) -#define NTP_SENT(F) F(8057, 30, -1, NULL) - -#define ARP_HA_FORMAT(F) F(8057, 31, 2, NULL) -#define ARP_PA_FORMAT(F) F(8057, 32, 2, NULL) -#define ARP_OPCODE(F) F(8057, 33, 2, NULL) -#define ARP_SRC_HA(F) F(8057, 34, -1, NULL) -#define ARP_SRC_PA(F) F(8057, 35, -1, NULL) -#define ARP_DST_HA(F) F(8057, 36, -1, NULL) -#define ARP_DST_PA(F) F(8057, 37, -1, NULL) - -#define TLS_SNI(F) F(8057, 808, -1, NULL) -#define TLS_ALPN(F) F(8057, 809, -1, NULL) -#define TLS_JA3(F) F(8057, 830, -1, NULL) - -#define SMTP_COMMANDS(F) F(8057, 810, 4, NULL) -#define SMTP_MAIL_COUNT(F) F(8057, 811, 4, NULL) -#define SMTP_RCPT_COUNT(F) F(8057, 812, 4, NULL) -#define SMTP_SENDER(F) F(8057, 813, -1, NULL) -#define SMTP_RECIPIENT(F) F(8057, 814, -1, NULL) -#define SMTP_STATUS_CODES(F) F(8057, 815, 4, NULL) -#define SMTP_CODE_2XX_COUNT(F) F(8057, 816, 4, NULL) -#define SMTP_CODE_3XX_COUNT(F) F(8057, 817, 4, NULL) -#define SMTP_CODE_4XX_COUNT(F) F(8057, 818, 4, NULL) -#define SMTP_CODE_5XX_COUNT(F) F(8057, 819, 4, NULL) -#define SMTP_DOMAIN(F) F(8057, 820, -1, NULL) - -#define SSDP_LOCATION_PORT(F) F(8057, 821, 2, NULL) -#define SSDP_SERVER(F) F(8057, 822, -1, NULL) -#define SSDP_USER_AGENT(F) F(8057, 823, -1, NULL) -#define SSDP_NT(F) F(8057, 824, -1, NULL) -#define SSDP_ST(F) F(8057, 825, -1, NULL) - -#define DNSSD_QUERIES(F) F(8057, 826, -1, NULL) -#define DNSSD_RESPONSES(F) F(8057, 827, -1, NULL) - -#define OVPN_CONF_LEVEL(F) F(8057, 828, 1, NULL) - -#define NB_NAME(F) F(8057, 831, -1, NULL) -#define NB_SUFFIX(F) F(8057, 832, 1, NULL) - - -#define IDP_CONTENT(F) F(8057, 850, -1, NULL) -#define IDP_CONTENT_REV(F) F(8057, 851, -1, NULL) - -#define STATS_PCKT_SIZES(F) F(0, 291, -1, NULL) // BASIC LIST -- FIELD IS e8057id1013 (uint16*) -#define STATS_PCKT_TIMESTAMPS(F) F(0, 291, -1, NULL) // BASIC LIST -- FIELD IS e8057id1014 (time*) -#define STATS_PCKT_TCPFLGS(F) F(0, 291, -1, NULL) // BASIC LIST -- FIELD IS e8057id1015 (uint8*) -#define STATS_PCKT_DIRECTIONS(F) F(0, 291, -1, NULL) // BASIC LIST -- FIELD IS e8057id1016 (int8*) - - -#define SBI_BRST_PACKETS(F) F(0, 291, -1, NULL) // BASIC LIST -- FIELD IS e8057id1050 (uint16*) -#define SBI_BRST_BYTES(F) F(0, 291, -1, NULL) // BASIC LIST -- FIELD IS e8057id1051 (uint16*) -#define SBI_BRST_TIME_START(F) F(0, 291, -1, NULL) // BASIC LIST -- FIELD IS e8057id1052 (time*) -#define SBI_BRST_TIME_STOP(F) F(0, 291, -1, NULL) // BASIC LIST -- FIELD IS e8057id1053 (time*) -#define DBI_BRST_PACKETS(F) F(0, 291, -1, NULL) // BASIC LIST -- FIELD IS e8057id1054 (uint16*) -#define DBI_BRST_BYTES(F) F(0, 291, -1, NULL) // BASIC LIST -- FIELD IS e8057id1055 (uint16*) -#define DBI_BRST_TIME_START(F) F(0, 291, -1, NULL) // BASIC LIST -- FIELD IS e8057id1056 (time*) -#define DBI_BRST_TIME_STOP(F) F(0, 291, -1, NULL) // BASIC LIST -- FIELD IS e8057id1057 (time*) - -#define D_PHISTS_IPT(F) F(0, 291, -1, NULL) // BASIC LIST -- FIELD IS e8057id1063 (uint16*) -#define D_PHISTS_SIZES(F) F(0, 291, -1, NULL) // BASIC LIST -- FIELD IS e8057id1062 (uint16*) -#define S_PHISTS_SIZES(F) F(0, 291, -1, NULL) // BASIC LIST -- FIELD IS e8057id1060 (uint16*) -#define S_PHISTS_IPT(F) F(0, 291, -1, NULL) // BASIC LIST -- FIELD IS e8057id1061 (uint16*) - -#define WG_CONF_LEVEL(F) F(8057, 861, 1, NULL) -#define WG_SRC_PEER(F) F(8057, 862, 4, NULL) -#define WG_DST_PEER(F) F(8057, 863, 4, NULL) - -/** - * IPFIX Templates - list of elements - * - * Each template is defined as a macro function expecting one argument F. - * This argument must be a macro function which is substituted with every - * specified element of the template. - * - * For instance, BASIC_TMPLT_V4 contains FLOW_END_REASON, BYTES, BYTES_REV, PACKETS,... - * all of them defined above. - */ - -#define BASIC_TMPLT_V4(F) \ - F(FLOW_END_REASON) \ - F(BYTES) \ - F(BYTES_REV) \ - F(PACKETS) \ - F(PACKETS_REV) \ - F(FLOW_START_USEC) \ - F(FLOW_END_USEC) \ - F(L3_PROTO) \ - F(L4_PROTO) \ - F(L4_TCP_FLAGS) \ - F(L4_TCP_FLAGS_REV) \ - F(L4_PORT_SRC) \ - F(L4_PORT_DST) \ - F(INPUT_INTERFACE) \ - F(L3_IPV4_ADDR_SRC) \ - F(L3_IPV4_ADDR_DST) \ - F(L2_SRC_MAC) \ - F(L2_DST_MAC) - -#define BASIC_TMPLT_V6(F) \ - F(FLOW_END_REASON) \ - F(BYTES) \ - F(BYTES_REV) \ - F(PACKETS) \ - F(PACKETS_REV) \ - F(FLOW_START_USEC) \ - F(FLOW_END_USEC) \ - F(L3_PROTO) \ - F(L4_PROTO) \ - F(L4_TCP_FLAGS) \ - F(L4_TCP_FLAGS_REV) \ - F(L4_PORT_SRC) \ - F(L4_PORT_DST) \ - F(INPUT_INTERFACE) \ - F(L3_IPV6_ADDR_SRC) \ - F(L3_IPV6_ADDR_DST) \ - F(L2_SRC_MAC) \ - F(L2_DST_MAC) - -#define IPFIX_HTTP_TEMPLATE(F) \ - F(HTTP_USERAGENT) \ - F(HTTP_METHOD) \ - F(HTTP_DOMAIN) \ - F(HTTP_REFERER) \ - F(HTTP_URI) \ - F(HTTP_CONTENT_TYPE) \ - F(HTTP_STATUS) - -#define IPFIX_RTSP_TEMPLATE(F) \ - F(RTSP_METHOD) \ - F(RTSP_USERAGENT) \ - F(RTSP_URI) \ - F(RTSP_STATUS)\ - F(RTSP_SERVER) \ - F(RTSP_CONTENT_TYPE) - -#define IPFIX_TLS_TEMPLATE(F) \ - F(TLS_SNI)\ - F(TLS_ALPN)\ - F(TLS_JA3) - -#define IPFIX_NTP_TEMPLATE(F) \ - F(NTP_LEAP) \ - F(NTP_VERSION) \ - F(NTP_MODE) \ - F(NTP_STRATUM) \ - F(NTP_POLL) \ - F(NTP_PRECISION) \ - F(NTP_DELAY) \ - F(NTP_DISPERSION) \ - F(NTP_REF_ID) \ - F(NTP_REF) \ - F(NTP_ORIG) \ - F(NTP_RECV) \ - F(NTP_SENT) - -#define IPFIX_DNS_TEMPLATE(F) \ - F(DNS_ANSWERS) \ - F(DNS_RCODE) \ - F(DNS_QTYPE) \ - F(DNS_CLASS) \ - F(DNS_RR_TTL) \ - F(DNS_RLENGTH) \ - F(DNS_PSIZE) \ - F(DNS_DO) \ - F(DNS_ID) \ - F(DNS_NAME) \ - F(DNS_RDATA) - -#define IPFIX_PASSIVEDNS_TEMPLATE(F) \ - F(DNS_ID) \ - F(DNS_RR_TTL) \ - F(DNS_ATYPE) \ - F(DNS_RDATA) \ - F(DNS_NAME) - -#define IPFIX_SMTP_TEMPLATE(F) \ - F(SMTP_COMMANDS) \ - F(SMTP_MAIL_COUNT) \ - F(SMTP_RCPT_COUNT) \ - F(SMTP_STATUS_CODES) \ - F(SMTP_CODE_2XX_COUNT) \ - F(SMTP_CODE_3XX_COUNT) \ - F(SMTP_CODE_4XX_COUNT) \ - F(SMTP_CODE_5XX_COUNT) \ - F(SMTP_DOMAIN) \ - F(SMTP_SENDER) \ - F(SMTP_RECIPIENT) - -#define IPFIX_SIP_TEMPLATE(F) \ - F(SIP_MSG_TYPE) \ - F(SIP_STATUS_CODE) \ - F(SIP_CSEQ) \ - F(SIP_CALLING_PARTY) \ - F(SIP_CALLED_PARTY) \ - F(SIP_CALL_ID) \ - F(SIP_USER_AGENT) \ - F(SIP_REQUEST_URI) \ - F(SIP_VIA) - -#define IPFIX_PSTATS_TEMPLATE(F) \ - F(STATS_PCKT_SIZES) \ - F(STATS_PCKT_TIMESTAMPS) \ - F(STATS_PCKT_TCPFLGS) \ - F(STATS_PCKT_DIRECTIONS) - -#define IPFIX_OVPN_TEMPLATE(F) \ - F(OVPN_CONF_LEVEL) - -#define IPFIX_SSDP_TEMPLATE(F) \ - F(SSDP_LOCATION_PORT) \ - F(SSDP_NT) \ - F(SSDP_USER_AGENT)\ - F(SSDP_ST) \ - F(SSDP_SERVER) - -#define IPFIX_DNSSD_TEMPLATE(F) \ - F(DNSSD_QUERIES) \ - F(DNSSD_RESPONSES) - -#define IPFIX_IDPCONTENT_TEMPLATE(F) \ - F(IDP_CONTENT) \ - F(IDP_CONTENT_REV) - -#define IPFIX_BSTATS_TEMPLATE(F) \ - F(SBI_BRST_PACKETS) \ - F(SBI_BRST_BYTES) \ - F(SBI_BRST_TIME_START) \ - F(SBI_BRST_TIME_STOP) \ - F(DBI_BRST_PACKETS) \ - F(DBI_BRST_BYTES) \ - F(DBI_BRST_TIME_START) \ - F(DBI_BRST_TIME_STOP) - -#define IPFIX_NETBIOS_TEMPLATE(F) \ - F(NB_SUFFIX) \ - F(NB_NAME) - -#define IPFIX_NETBIOS_TEMPLATE(F) \ - F(NB_SUFFIX) \ - F(NB_NAME) - -#define IPFIX_BASICPLUS_TEMPLATE(F) \ - F(L3_TTL) \ - F(L3_TTL_REV) \ - F(L3_FLAGS) \ - F(L3_FLAGS_REV) \ - F(L4_TCP_WIN) \ - F(L4_TCP_WIN_REV) \ - F(L4_TCP_OPTIONS) \ - F(L4_TCP_OPTIONS_REV) \ - F(L4_TCP_MSS) \ - F(L4_TCP_MSS_REV) \ - F(L4_TCP_SYN_SIZE) - -#define IPFIX_PHISTS_TEMPLATE(F) \ - F(S_PHISTS_SIZES) \ - F(S_PHISTS_IPT) \ - F(D_PHISTS_SIZES) \ - F(D_PHISTS_IPT) - -#define IPFIX_WG_TEMPLATE(F) \ - F(WG_CONF_LEVEL) \ - F(WG_SRC_PEER) \ - F(WG_DST_PEER) - -/** - * List of all known templated. - * - * This macro is define in order to use all elements of all defined - * templates at once. - */ -#define IPFIX_ENABLED_TEMPLATES(F) \ - BASIC_TMPLT_V4(F) \ - BASIC_TMPLT_V6(F) \ - IPFIX_HTTP_TEMPLATE(F) \ - IPFIX_RTSP_TEMPLATE(F) \ - IPFIX_TLS_TEMPLATE(F) \ - IPFIX_NTP_TEMPLATE(F) \ - IPFIX_SIP_TEMPLATE(F) \ - IPFIX_DNS_TEMPLATE(F) \ - IPFIX_PASSIVEDNS_TEMPLATE(F) \ - IPFIX_PSTATS_TEMPLATE(F) \ - IPFIX_OVPN_TEMPLATE(F) \ - IPFIX_SMTP_TEMPLATE(F) \ - IPFIX_SSDP_TEMPLATE(F) \ - IPFIX_DNSSD_TEMPLATE(F) \ - IPFIX_IDPCONTENT_TEMPLATE(F) \ - IPFIX_NETBIOS_TEMPLATE(F) \ - IPFIX_BASICPLUS_TEMPLATE(F) \ - IPFIX_BSTATS_TEMPLATE(F) \ - IPFIX_PHISTS_TEMPLATE(F) \ - IPFIX_WG_TEMPLATE(F) - - - -/** - * Helper macro, convert FIELD into its name as a C literal. - * - * For instance, processing: IPFIX_FIELD_NAMES(BYTES) with C-preprocessor - * produces "BYTES". - */ -#define IPFIX_FIELD_NAMES(F) #F, - -#endif diff --git a/process/osquery.cpp b/process/osquery.cpp new file mode 100644 index 00000000..2caf75a2 --- /dev/null +++ b/process/osquery.cpp @@ -0,0 +1,652 @@ +/** + * \file osqueryplugin.cpp + * \brief Plugin for parsing osquery traffic. + * \author Anton Aheyeu aheyeant@fit.cvut.cz + * \date 2021 + */ + +/* + * Copyright (C) 2021 CESNET + * + * LICENSE TERMS + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided as is'', and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#include +#include +#include +#include +#include + +#include "osquery.hpp" + +#define HEX(x) std::setw(2) << std::setfill('0') << std::hex << (int) (x) + +namespace ipxp { + +int RecordExtOSQUERY::REGISTERED_ID = -1; + +__attribute__((constructor)) static void register_this_plugin() +{ + static PluginRecord rec = PluginRecord("osquery", [](){return new OSQUERYPlugin();}); + register_plugin(&rec); + RecordExtOSQUERY::REGISTERED_ID = register_extension(); +} + +OSQUERYPlugin::OSQUERYPlugin() : manager(nullptr), numberOfSuccessfullyRequests(0) +{ +} + +OSQUERYPlugin::OSQUERYPlugin(const OSQUERYPlugin &p) +{ + init(nullptr); +} + +OSQUERYPlugin::~OSQUERYPlugin() +{ + close(); +} + +void OSQUERYPlugin::init(const char *params) +{ + manager = new OsqueryRequestManager(); + manager->readInfoAboutOS(); +} + +void OSQUERYPlugin::close() +{ + if (manager != nullptr) { + delete manager; + manager = nullptr; + } +} + +ProcessPlugin *OSQUERYPlugin::copy() +{ + return new OSQUERYPlugin(*this); +} + +int OSQUERYPlugin::post_create(Flow &rec, const Packet &pkt) +{ + ConvertedFlowData flowDataIPv4(rec.src_ip.v4, rec.dst_ip.v4, rec.src_port, rec.dst_port); + + if (manager->readInfoAboutProgram(flowDataIPv4)) { + RecordExtOSQUERY *record = new RecordExtOSQUERY(manager->getRecord()); + rec.add_extension(record); + + numberOfSuccessfullyRequests++; + } + + return 0; +} + +void OSQUERYPlugin::finish(bool print_stats) +{ + if (print_stats) { + std::cout << "OSQUERY plugin stats:" << std::endl; + std::cout << "Number of successfully processed requests: " << numberOfSuccessfullyRequests << std::endl; + } +} + +ConvertedFlowData::ConvertedFlowData(uint32_t sourceIPv4, uint32_t destinationIPv4, uint16_t sourcePort, + uint16_t destinationPort) +{ + convertIPv4(sourceIPv4, true); + convertIPv4(destinationIPv4, false); + convertPort(sourcePort, true); + convertPort(destinationPort, false); +} + +ConvertedFlowData::ConvertedFlowData(const uint8_t *sourceIPv6, const uint8_t *destinationIPv6, uint16_t sourcePort, + uint16_t destinationPort) +{ + convertIPv6(sourceIPv6, true); + convertIPv6(destinationIPv6, false); + convertPort(sourcePort, true); + convertPort(destinationPort, false); +} + +void ConvertedFlowData::convertIPv4(uint32_t addr, bool isSourceIP) +{ + std::stringstream ss; + + ss << ((addr) & 0x000000ff) << "." + << ((addr >> 8) & 0x000000ff) << "." + << ((addr >> 16) & 0x000000ff) << "." + << ((addr >> 24) & 0x000000ff); + + if (isSourceIP) { + this->src_ip = ss.str(); + } else { + this->dst_ip = ss.str(); + } +} + +void ConvertedFlowData::convertIPv6(const uint8_t *addr, bool isSourceIP) +{ + std::stringstream ss; + + ss << HEX(addr[0]); + for (int i = 1; i < 16; i++) { + ss << ":" << HEX(addr[i]); + } + + if (isSourceIP) { + this->src_ip = ss.str(); + } else { + this->dst_ip = ss.str(); + } +} + +void ConvertedFlowData::convertPort(uint16_t port, bool isSourcePort) +{ + std::stringstream ss; + + ss << port; + + if (isSourcePort) { + this->src_port = ss.str(); + } else { + this->dst_port = ss.str(); + } +} + +OsqueryRequestManager::OsqueryRequestManager() : + inputFD(0), + outputFD(0), + buffer(nullptr), + pfd(nullptr), + recOsquery(nullptr), + isFDOpened(false), + numberOfAttempts(0), + osqueryProcessId(-1) +{ + buffer = new char [BUFFER_SIZE]; + + pfd = new pollfd; + pfd->events = POLLIN; + + recOsquery = new RecordExtOSQUERY(); + + while (true) { + openOsqueryFD(); + if (handler.isFatalError()) { + break; + } else if (handler.isOpenFDError()) { + continue; + } else { + break; + } + } +} + +OsqueryRequestManager::~OsqueryRequestManager() +{ + delete[] buffer; + delete pfd; + delete recOsquery; + closeOsqueryFD(); +} + +void OsqueryRequestManager::readInfoAboutOS() +{ + const std::string query = + "SELECT ov.name, ov.major, ov.minor, ov.build, ov.platform, ov.platform_like, ov.arch, ki.version, si.hostname FROM os_version AS ov, kernel_info AS ki, system_info AS si;\r\n"; + + if (executeQuery(query) > 0) { + parseJsonOSVersion(); + } +} + +bool OsqueryRequestManager::readInfoAboutProgram(const ConvertedFlowData &flowData) +{ + if (handler.isFatalError()) { + return false; + } + + recOsquery->program_name = DEFAULT_FILL_TEXT; + recOsquery->username = DEFAULT_FILL_TEXT; + + std::string pid; + + if (!getPID(pid, flowData)) { + return false; + } + + std::string query = "SELECT p.name, u.username FROM processes AS p INNER JOIN users AS u ON p.uid=u.uid " + "WHERE p.pid='" + pid + "';\r\n"; + + if (executeQuery(query) > 0) { + if (parseJsonAboutProgram()) { + return true; + } + } + return false; +} + +size_t OsqueryRequestManager::executeQuery(const std::string &query, bool reopenFD) +{ + if (reopenFD) { + openOsqueryFD(); + } + + if (handler.isFatalError()) { + return 0; + } + + if (handler.isOpenFDError()) { + return executeQuery(query, true); + } + + handler.refresh(); + + if (!writeToOsquery(query.c_str())) { + return executeQuery(query, true); + } + + size_t ret = readFromOsquery(); + + if (handler.isReadError()) { + return executeQuery(query, true); + } + + if (handler.isReadSuccess()) { + numberOfAttempts = 0; + return ret; + } + + return 0; +} + +bool OsqueryRequestManager::writeToOsquery(const char *query) +{ + // If expression is true, a logical error has occurred. + // There should be no logged errors when executing this method + if (handler.isErrorState()) { + handler.setFatalError(); + return false; + } + + ssize_t length = strlen(query); + ssize_t n = write(inputFD, query, length); + + return (n != -1 && n == length); +} + +size_t OsqueryRequestManager::readFromOsquery() +{ + // If expression is true, a logical error has occurred. + // There should be no logged errors when executing this method + if (handler.isErrorState()) { + handler.setFatalError(); + return 0; + } + + clearBuffer(); + pfd->revents = 0; + + int ret = poll(pfd, 1, POLL_TIMEOUT); + + // ret == -1 -> poll error. + // ret == 0 -> poll timeout (osquery in json mode always returns at least empty json string("[\n\n]\n"), + // if no response has been received, this is considered an error). + if (ret == -1 || ret == 0) { + handler.setReadError(); + return 0; + } + + if (pfd->revents & POLLIN) { + size_t bytesRead = 0; + while (true) { + if (bytesRead + READ_SIZE < BUFFER_SIZE) { + ssize_t n = read(outputFD, buffer + bytesRead, READ_SIZE); + + // read error + if (n < 0) { + handler.setReadError(); + return 0; + } + + bytesRead += n; + + // Error: less than 5 bytes were read + if (bytesRead < 5) { + clearBuffer(); + handler.setReadError(); + return 0; + } + + if (n < READ_SIZE || buffer[bytesRead - 2] == ']') { + buffer[bytesRead] = 0; + handler.setReadSuccess(); + return bytesRead; + } + } else { + ssize_t n = read(outputFD, buffer, READ_SIZE); + + // read error + if (n < 0) { + handler.setReadError(); + return 0; + } + + if (n < READ_SIZE || buffer[n - 2] == ']') { + clearBuffer(); + handler.setReadSuccess(); + return 0; + } + } + } + } + handler.setReadError(); + return 0; +} // OsqueryRequestManager::readFromOsquery + +void OsqueryRequestManager::openOsqueryFD() +{ + if (handler.isFatalError()) { + return; + } + + // All attempts have been exhausted + if (numberOfAttempts >= MAX_NUMBER_OF_ATTEMPTS) { + handler.setFatalError(); + return; + } + + closeOsqueryFD(); + killPreviousProcesses(); + handler.reset(); + numberOfAttempts++; + + osqueryProcessId = popen2("osqueryi --json 2>/dev/null", &inputFD, &outputFD); + + if (osqueryProcessId <= 0) { + handler.setOpenFDError(); + return; + } else { + isFDOpened = true; + pfd->fd = outputFD; + return; + } +} + +void OsqueryRequestManager::closeOsqueryFD() +{ + if (isFDOpened) { + close(inputFD); + close(outputFD); + isFDOpened = false; + } +} + +void OsqueryRequestManager::killPreviousProcesses(bool useWhonangOption) const +{ + if (useWhonangOption) { + waitpid(-1, nullptr, WNOHANG); + } else { + if (osqueryProcessId > 0) { + waitpid(osqueryProcessId, nullptr, 0); + } + } +} + +bool OsqueryRequestManager::getPID(std::string &pid, const ConvertedFlowData &flowData) +{ + std::string query = "SELECT pid FROM process_open_sockets WHERE " + "(local_address='" + flowData.src_ip + "' AND " + "remote_address='" + flowData.dst_ip + "' AND " + "local_port='" + flowData.src_port + "' AND " + "remote_port='" + flowData.dst_port + "') OR " + "(local_address='" + flowData.dst_ip + "' AND " + "remote_address='" + flowData.src_ip + "' AND " + "local_port='" + flowData.dst_port + "' AND " + "remote_port='" + flowData.src_port + "') LIMIT 1;\r\n"; + + if (executeQuery(query) > 0) { + if (parseJsonSingleItem("pid", pid)) { + return true; + } + } + + return false; +} + +bool OsqueryRequestManager::parseJsonSingleItem(const std::string &singleKey, std::string &singleValue) +{ + int pos = getPositionForParseJson(); + + if (pos == -1) { + return false; + } + + int count = 0; + std::string key, value; + while (true) { + key.clear(); + value.clear(); + pos = parseJsonItem(pos, key, value); + if (pos < 0) { + return false; + } + if (pos == 0) { + return count == 1; + } + + if (key == singleKey) { + singleValue = value; + count++; + } else { + return false; + } + } +} + +bool OsqueryRequestManager::parseJsonOSVersion() +{ + int pos = getPositionForParseJson(); + + if (pos == -1) { + return false; + } + + int count = 0; + std::string key, value; + + while (true) { + key.clear(); + value.clear(); + pos = parseJsonItem(pos, key, value); + if (pos < 0) { + return false; + } + if (pos == 0) { + return count == 9; + } + if (key == "arch") { + recOsquery->os_arch = std::string(value); + count++; + } else if (key == "build") { + recOsquery->os_build = value; + count++; + } else if (key == "hostname") { + recOsquery->system_hostname = value; + count++; + } else if (key == "major") { + recOsquery->os_major = atoi(value.c_str()); + count++; + } else if (key == "minor") { + recOsquery->os_minor = atoi(value.c_str()); + count++; + } else if (key == "name") { + recOsquery->os_name = value; + count++; + } else if (key == "platform") { + recOsquery->os_platform = value; + count++; + } else if (key == "platform_like") { + recOsquery->os_platform_like = value; + count++; + } else if (key == "version") { + recOsquery->kernel_version = value; + count++; + } else { + return false; + } + } +} // OsqueryRequestManager::parseJsonOSVersion + +bool OsqueryRequestManager::parseJsonAboutProgram() +{ + int pos = getPositionForParseJson(); + + if (pos == -1) { + return false; + } + + int count = 0; + std::string key, value; + + while (true) { + key.clear(); + value.clear(); + pos = parseJsonItem(pos, key, value); + if (pos < 0) { + return false; + } + if (pos == 0) { + return count == 2; + } + + if (key == "name") { + recOsquery->program_name = value; + count++; + } else if (key == "username") { + recOsquery->username = value; + count++; + } else { + return false; + } + } +} + +int OsqueryRequestManager::parseJsonItem(int from, std::string &key, std::string &value) +{ + int pos = parseString(from, key); + + if (pos < 0) { + return -1; + } + if (pos == 0) { + return 0; + } + if (buffer[pos] != ':') { + return -1; + } + + pos = parseString(pos, value); + if (pos <= 0) { + return -1; + } + return pos; +} + +int OsqueryRequestManager::parseString(int from, std::string &str) +{ + int pos = from; + bool findQuotes = false; + char c; + + while (true) { + c = buffer[pos]; + pos++; + if (c == 0) { + return -1; + } else if (c == '}') { + return 0; + } else if (c == '\"') { + if (!findQuotes) { + findQuotes = true; + } else { + break; + } + } else if (findQuotes) { + str += c; + } + } + return pos; +} + +pid_t OsqueryRequestManager::popen2(const char *command, int *inFD, int *outFD) const +{ + int p_stdin[2], p_stdout[2]; + pid_t pid; + + if (pipe(p_stdin) != 0 || pipe(p_stdout) != 0) { + return -1; + } + + pid = fork(); + + if (pid < 0) { + return pid; + } else if (pid == 0) { + close(p_stdin[WRITE_FD]); + dup2(p_stdin[READ_FD], READ_FD); + close(p_stdout[READ_FD]); + dup2(p_stdout[WRITE_FD], WRITE_FD); + execl("/bin/sh", "sh", "-c", command, nullptr); + perror("execl"); + exit(1); + } + + inFD == nullptr ? close(p_stdin[WRITE_FD]) : *inFD = p_stdin[WRITE_FD]; + outFD == nullptr ? close(p_stdout[READ_FD]) : *outFD = p_stdout[READ_FD]; + + return pid; +} + +int OsqueryRequestManager::getPositionForParseJson() +{ + int position = 0; + + while (buffer[position] != 0) { + if (buffer[position] == '[') { + return position + 1; + } + position++; + } + return -1; +} + +} diff --git a/process/osquery.hpp b/process/osquery.hpp new file mode 100644 index 00000000..3685908b --- /dev/null +++ b/process/osquery.hpp @@ -0,0 +1,559 @@ +/** + * \file osqueryplugin.hpp + * \brief Plugin for parsing osquery traffic. + * \author Anton Aheyeu aheyeant@fit.cvut.cz + * \date 2021 + */ + +/* + * Copyright (C) 2021 CESNET + * + * LICENSE TERMS + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * 3. Neither the name of the Company nor the names of its contributors + * may be used to endorse or promote products derived from this + * software without specific prior written permission. + * + * ALTERNATIVELY, provided that this notice is retained in full, this + * product may be distributed under the terms of the GNU General Public + * License (GPL) version 2 or later, in which case the provisions + * of the GPL apply INSTEAD OF those given above. + * + * This software is provided as is'', and any express or implied + * warranties, including, but not limited to, the implied warranties of + * merchantability and fitness for a particular purpose are disclaimed. + * In no event shall the company or contributors be liable for any + * direct, indirect, incidental, special, exemplary, or consequential + * damages (including, but not limited to, procurement of substitute + * goods or services; loss of use, data, or profits; or business + * interruption) however caused and on any theory of liability, whether + * in contract, strict liability, or tort (including negligence or + * otherwise) arising in any way out of the use of this software, even + * if advised of the possibility of such damage. + * + */ + +#ifndef IPXP_PROCESS_OSQUERY_HPP +#define IPXP_PROCESS_OSQUERY_HPP + +#include +#include +#include +#include +#include + +#ifdef WITH_NEMEA +# include "fields.h" +#endif + +#include +#include +#include +#include + +#define DEFAULT_FILL_TEXT "UNDEFINED" + +// OsqueryStateHandler +#define FATAL_ERROR 0b00000001 // 1; Fatal error, cannot be fixed +#define OPEN_FD_ERROR 0b00000010 // 2; Failed to open osquery FD +#define READ_ERROR 0b00000100 // 4; Error while reading +#define READ_SUCCESS 0b00001000 // 8; Data read successfully + +// OsqueryRequestManager +#define BUFFER_SIZE 1024 * 20 + 1 +#define READ_SIZE 1024 +#define POLL_TIMEOUT 200 // millis +#define READ_FD 0 +#define WRITE_FD 1 +#define MAX_NUMBER_OF_ATTEMPTS 2 // Max number of osquery error correction attempts + +#define OSQUERY_UNIREC_TEMPLATE \ + "OSQUERY_PROGRAM_NAME,OSQUERY_USERNAME,OSQUERY_OS_NAME,OSQUERY_OS_MAJOR,OSQUERY_OS_MINOR,OSQUERY_OS_BUILD,OSQUERY_OS_PLATFORM,OSQUERY_OS_PLATFORM_LIKE,OSQUERY_OS_ARCH,OSQUERY_KERNEL_VERSION,OSQUERY_SYSTEM_HOSTNAME" + +UR_FIELDS( + string OSQUERY_PROGRAM_NAME, + string OSQUERY_USERNAME, + string OSQUERY_OS_NAME, + uint16 OSQUERY_OS_MAJOR, + uint16 OSQUERY_OS_MINOR, + string OSQUERY_OS_BUILD, + string OSQUERY_OS_PLATFORM, + string OSQUERY_OS_PLATFORM_LIKE, + string OSQUERY_OS_ARCH, + string OSQUERY_KERNEL_VERSION, + string OSQUERY_SYSTEM_HOSTNAME +) + +namespace ipxp { + +/** + * \brief Flow record extension header for storing parsed OSQUERY packets. + */ +struct RecordExtOSQUERY : public RecordExt { + static int REGISTERED_ID; + std::string program_name; + std::string username; + std::string os_name; + uint16_t os_major; + uint16_t os_minor; + std::string os_build; + std::string os_platform; + std::string os_platform_like; + std::string os_arch; + std::string kernel_version; + std::string system_hostname; + + + RecordExtOSQUERY() : RecordExt(REGISTERED_ID) + { + program_name = DEFAULT_FILL_TEXT; + username = DEFAULT_FILL_TEXT; + os_name = DEFAULT_FILL_TEXT; + os_major = 0; + os_minor = 0; + os_build = DEFAULT_FILL_TEXT; + os_platform = DEFAULT_FILL_TEXT; + os_platform_like = DEFAULT_FILL_TEXT; + os_arch = DEFAULT_FILL_TEXT; + kernel_version = DEFAULT_FILL_TEXT; + system_hostname = DEFAULT_FILL_TEXT; + } + + RecordExtOSQUERY(const RecordExtOSQUERY *record) : RecordExt(REGISTERED_ID) + { + program_name = record->program_name; + username = record->username; + os_name = record->os_name; + os_major = record->os_major; + os_minor = record->os_minor; + os_build = record->os_build; + os_platform = record->os_platform; + os_platform_like = record->os_platform_like; + os_arch = record->os_arch; + kernel_version = record->kernel_version; + system_hostname = record->system_hostname; + } + + #ifdef WITH_NEMEA + virtual void fillUnirec(ur_template_t *tmplt, void *record) + { + ur_set_string(tmplt, record, F_OSQUERY_PROGRAM_NAME, program_name.c_str()); + ur_set_string(tmplt, record, F_OSQUERY_USERNAME, username.c_str()); + ur_set_string(tmplt, record, F_OSQUERY_OS_NAME, os_name.c_str()); + ur_set(tmplt, record, F_OSQUERY_OS_MAJOR, os_major); + ur_set(tmplt, record, F_OSQUERY_OS_MINOR, os_minor); + ur_set_string(tmplt, record, F_OSQUERY_OS_BUILD, os_build.c_str()); + ur_set_string(tmplt, record, F_OSQUERY_OS_PLATFORM, os_platform.c_str()); + ur_set_string(tmplt, record, F_OSQUERY_OS_PLATFORM_LIKE, os_platform_like.c_str()); + ur_set_string(tmplt, record, F_OSQUERY_OS_ARCH, os_arch.c_str()); + ur_set_string(tmplt, record, F_OSQUERY_KERNEL_VERSION, kernel_version.c_str()); + ur_set_string(tmplt, record, F_OSQUERY_SYSTEM_HOSTNAME, system_hostname.c_str()); + } + + const char *get_unirec_tmplt() const + { + return OSQUERY_UNIREC_TEMPLATE; + } + #endif // ifdef WITH_NEMEA + + virtual int fillIPFIX(uint8_t *buffer, int size) + { + int length, total_length = 0; + + // OSQUERY_PROGRAM_NAME + length = program_name.length(); + if (total_length + length + 1 > size) { + return -1; + } + buffer[total_length] = length; + memcpy(buffer + total_length + 1, program_name.c_str(), length); + total_length += length + 1; + + // OSQUERY_USERNAME + length = username.length(); + if (total_length + length + 1 > size) { + return -1; + } + buffer[total_length] = length; + memcpy(buffer + total_length + 1, username.c_str(), length); + total_length += length + 1; + + // OSQUERY_OS_NAME + length = os_name.length(); + if (total_length + length + 1 > size) { + return -1; + } + buffer[total_length] = length; + memcpy(buffer + total_length + 1, os_name.c_str(), length); + total_length += length + 1; + + // OSQUERY_OS_MAJOR + *(uint16_t *) (buffer + total_length) = ntohs(os_major); + total_length += 2; + + // OSQUERY_OS_MINOR + *(uint16_t *) (buffer + total_length) = ntohs(os_minor); + total_length += 2; + + // OSQUERY_OS_BUILD + length = os_build.length(); + if (total_length + length + 1 > size) { + return -1; + } + buffer[total_length] = length; + memcpy(buffer + total_length + 1, os_build.c_str(), length); + total_length += length + 1; + + // OSQUERY_OS_PLATFORM + length = os_platform.length(); + if (total_length + length + 1 > size) { + return -1; + } + buffer[total_length] = length; + memcpy(buffer + total_length + 1, os_platform.c_str(), length); + total_length += length + 1; + + // OSQUERY_OS_PLATFORM_LIKE + length = os_platform_like.length(); + if (total_length + length + 1 > size) { + return -1; + } + buffer[total_length] = length; + memcpy(buffer + total_length + 1, os_platform_like.c_str(), length); + total_length += length + 1; + + // OSQUERY_OS_ARCH + length = os_arch.length(); + if (total_length + length + 1 > size) { + return -1; + } + buffer[total_length] = length; + memcpy(buffer + total_length + 1, os_arch.c_str(), length); + total_length += length + 1; + + // OSQUERY_KERNEL_VERSION + length = kernel_version.length(); + if (total_length + length + 1 > size) { + return -1; + } + buffer[total_length] = length; + memcpy(buffer + total_length + 1, kernel_version.c_str(), length); + total_length += length + 1; + + // OSQUERY_SYSTEM_HOSTNAME + length = system_hostname.length(); + if (total_length + length + 1 > size) { + return -1; + } + buffer[total_length] = length; + memcpy(buffer + total_length + 1, system_hostname.c_str(), length); + total_length += length + 1; + + return total_length; + } // fillIPFIX + + const char **get_ipfix_tmplt() const + { + static const char *ipfix_template[] = { + IPFIX_OSQUERY_TEMPLATE(IPFIX_FIELD_NAMES) + nullptr + }; + + return ipfix_template; + } + + std::string get_text() const + { + std::ostringstream out; + out << "program=\"" << program_name << "\"" + << ",username=\"" << username << "\"" + << ",osname=\"" << os_name << "\"" + << ",major=" << os_major + << ",minor=" << os_minor + << ",build=\"" << os_build << "\"" + << ",platform=\"" << os_platform << "\"" + << ",arch=\"" << os_arch << "\"" + << ",kernel=\"" << kernel_version << "\"" + << ",hostname=\"" << system_hostname << "\""; + return out.str(); + } +}; + + +/** + * \brief Additional structure for handling osquery states. + */ +struct OsqueryStateHandler { + OsqueryStateHandler() : OSQUERY_STATE(0){ } + + bool isErrorState() const { return (OSQUERY_STATE & (FATAL_ERROR | OPEN_FD_ERROR | READ_ERROR)); } + + void setFatalError(){ OSQUERY_STATE |= FATAL_ERROR; } + + bool isFatalError() const { return (OSQUERY_STATE & FATAL_ERROR); } + + void setOpenFDError(){ OSQUERY_STATE |= OPEN_FD_ERROR; } + + bool isOpenFDError() const { return (OSQUERY_STATE & OPEN_FD_ERROR); } + + void setReadError(){ OSQUERY_STATE |= READ_ERROR; } + + bool isReadError() const { return (OSQUERY_STATE & READ_ERROR); } + + void setReadSuccess(){ OSQUERY_STATE |= READ_SUCCESS; } + + bool isReadSuccess() const { return (OSQUERY_STATE & READ_SUCCESS); } + + /** + * Reset the \p OSQUERY_STATE. Fatal and open fd errors will not be reset. + */ + void refresh(){ OSQUERY_STATE = OSQUERY_STATE & (FATAL_ERROR | OPEN_FD_ERROR); } + + /** + * Reset the \p OSQUERY_STATE. Fatal and open fd errors will be reset. + */ + void reset(){ OSQUERY_STATE = 0; } + +private: + uint8_t OSQUERY_STATE; +}; + + +/** + * \brief Additional structure for store and convert data from flow (src_ip, dst_ip, src_port, dst_port) to string. + */ +struct ConvertedFlowData { + /** + * Constructor for IPv4-based flow. + * @param sourceIPv4 source IPv4 address. + * @param destinationIPv4 destination IPv4 address. + * @param sourcePort source port. + * @param destinationPort destination port. + */ + ConvertedFlowData(uint32_t sourceIPv4, uint32_t destinationIPv4, uint16_t sourcePort, uint16_t destinationPort); + + /** + * Constructor for IPv6-based flow. + * @param sourceIPv6 source IPv6 address. + * @param destinationIPv6 destination IPv6 address. + * @param sourcePort source port. + * @param destinationPort destination port. + */ + ConvertedFlowData(const uint8_t *sourceIPv6, const uint8_t *destinationIPv6, uint16_t sourcePort, + uint16_t destinationPort); + + std::string src_ip; + std::string dst_ip; + std::string src_port; + std::string dst_port; + +private: + + /** + * Converts an IPv4 numeric value to a string. + * @param addr IPv4 address. + * @param isSourceIP if true - source IP conversion mode, if false - destination IP conversion mode. + */ + void convertIPv4(uint32_t addr, bool isSourceIP); + + /** + * Converts an IPv6 numeric value to a string. + * @param addr IPv6 address. + * @param isSourceIP if true - source IP conversion mode, if false - destination IP conversion mode. + */ + void convertIPv6(const uint8_t *addr, bool isSourceIP); + + /** + * Converts the numeric port value to a string. + * @param port + * @param isSourcePort if true - source port conversion mode, if false - destination port conversion mode. + */ + void convertPort(uint16_t port, bool isSourcePort); +}; + + +/** + * \brief Manager for communication with osquery + */ +struct OsqueryRequestManager { + OsqueryRequestManager(); + + ~OsqueryRequestManager(); + + const RecordExtOSQUERY *getRecord(){ return recOsquery; } + + /** + * Fills the record with OS values from osquery. + */ + void readInfoAboutOS(); + + /** + * Fills the record with program values from osquery. + * @param flowData flow data converted to string. + * @return true if success or false. + */ + bool readInfoAboutProgram(const ConvertedFlowData &flowData); + +private: + + /** + * Sends a request and receives a response from osquery. + * @param query sql query according to osquery standards. + * @param reopenFD if true - tries to reopen fd. + * @return number of bytes read. + */ + size_t executeQuery(const std::string &query, bool reopenFD = false); + + /** + * Writes query to osquery input FD. + * @param query sql query according to osquery standards. + * @return true if success or false. + */ + bool writeToOsquery(const char *query); + + /** + * Reads data from osquery output FD. + * \note Can change osquery state. Possible changes: READ_ERROR, READ_SUCCESS. + * @return number of bytes read. + */ + size_t readFromOsquery(); + + /** + * Opens osquery FD. + * \note Can change osquery state. Possible changes: FATAL_ERROR, OPEN_FD_ERROR. + */ + void openOsqueryFD(); + + /** + * Closes osquery FD. + */ + void closeOsqueryFD(); + + /** + * Before reopening osquery tries to kill the previous osquery process. + * + * If \p useWhonangOption is true then the waitpid() function will be used + * in non-blocking mode(can be called before the process is ready to close, + * the process will remain in a zombie state). At the end of the application, + * a zombie process may remain, it will be killed when the application is closed. + * Else if \p useWhonangOption is false then the waitpid() function will be used + * in blocking mode(will wait for the process to complete). Will kill all unnecessary + * processes, but will block the application until the killed process is finished. + * + * @param useWhonangOption if true will be used non-blocking mode. + */ + void killPreviousProcesses(bool useWhonangOption = true) const; + + /** + * Tries to get the process id from table "process_open_sockets". + * @param[out] pid process id. + * @param[in] flowData flow data converted to string. + * @return true true if success or false. + */ + bool getPID(std::string &pid, const ConvertedFlowData &flowData); + + /** + * Parses json string with only one element. + * @param[in] singleKey key. + * @param[out] singleValue value. + * @return true if success or false. + */ + bool parseJsonSingleItem(const std::string &singleKey, std::string &singleValue); + + /** + * Parses json by template. + * @return true if success or false. + */ + bool parseJsonOSVersion(); + + /** + * Parses json by template. + * @return true if success or false. + */ + bool parseJsonAboutProgram(); + + /** + * From position \p from tries to find two strings between quotes ["key":"value"]. + * @param[in] from start position in the buffer. + * @param[out] key value for the "key" parsing result. + * @param[out] value value for the "value" parsing result. + * @return the position where the text search ended, 0 if end of json row or -1 if end of buffer. + */ + int parseJsonItem(int from, std::string &key, std::string &value); + + /** + * From position \p from tries to find string between quotes. + * @param[in] from start position in the buffer. + * @param[out] str value for the parsing result. + * @return the position where the text search ended, 0 if end of json row or -1 if end of buffer. + */ + int parseString(int from, std::string &str); + + /** + * Create a new process for connecting FD. + * @param[in] command command to execute in sh. + * @param[out] inFD input FD. + * @param[out] outFD output FD. + * @return pid of the new process. + */ + pid_t popen2(const char *command, int *inFD, int *outFD) const; + + /** + * Sets the first byte in the buffer to zero + */ + void clearBuffer(){ buffer[0] = 0; } + + /** + * Tries to find the position in the buffer where the json data starts. + * @return position number or -1 if position was not found. + */ + int getPositionForParseJson(); + + int inputFD; + int outputFD; + char * buffer; + pollfd * pfd; + RecordExtOSQUERY * recOsquery; + bool isFDOpened; + int numberOfAttempts; + pid_t osqueryProcessId; + + OsqueryStateHandler handler; +}; + + +/** + * \brief Flow cache plugin for parsing OSQUERY packets. + */ +class OSQUERYPlugin : public ProcessPlugin +{ +public: + OSQUERYPlugin(); + ~OSQUERYPlugin(); + OSQUERYPlugin(const OSQUERYPlugin &p); + void init(const char *params); + void close(); + RecordExt *get_ext() const { return new RecordExtOSQUERY(); } + OptionsParser *get_parser() const { return new OptionsParser("osquery", "Collect information about locally outbound flows from OS"); } + std::string get_name() const { return "osquery"; } + ProcessPlugin *copy(); + + int post_create(Flow &rec, const Packet &pkt); + void finish(bool print_stats); + +private: + OsqueryRequestManager *manager; + int numberOfSuccessfullyRequests; +}; + +} +#endif /* IPXP_PROCESS_OSQUERY_HPP */