diff --git a/eBPF_Supermarket/Network_Subsystem/tcp_watch/README.md b/eBPF_Supermarket/Network_Subsystem/tcp_watch/README.md index b7ff4a19f..0c0474697 100644 --- a/eBPF_Supermarket/Network_Subsystem/tcp_watch/README.md +++ b/eBPF_Supermarket/Network_Subsystem/tcp_watch/README.md @@ -1,17 +1,44 @@ +# eBPF-TCP-Watch +## 介绍 基于目前已有eBPF小工具，以及linux网络协议栈相关探测点，该项目在主机空间实现以下功能： +### 已完成 +- 搭建基础开发框架和自动编译管道。 +- 设计并实现TCP连接信息的记录 +- 设计并实现各个TCP连接发送与接收包信息的记录 +- 设计并实现TCP错误包信息的记录 +- 实现从TCP包中抽取HTTP信息并记录 +### TODO +- 增加运行时参数以提升可用性 +## 组织结构 +- tcpwatch.bpf.c：在各个内核探针点对TCP包信息、TCP连接信息以及各个包的HTTP1/1.1信息进行记录 +- tcpwatch.c: 对bpf.c文件中记录的信息进行输出 +- tcpwatch.h: 定义内核态与用户态程序共用的结构体 +- data/： + - connects.log：符合Prometheus格式的连接信息 + - err.log：符合Prometheus格式的错误包信息 + - packets.log：符合Prometheus格式的包信息 +- visual.py：暴露metrics接口给Prometheus，输出data文件夹下的所有信息 +## 快速开始 +### 安装依赖 +- OS: Ubuntu 22.04LTS +```bash +sudo apt update +sudo apt install libbpf-dev clang llvm libelf-dev libpcap-dev gcc-multilib build-essential +git submodule update --init --recursive +``` +### 编译运行 +```bash +make +sudo ./tcpwatch +``` +### 参数 +```bash +Usage: tcpwatch [OPTION...] +Watch tcp/ip in network subsystem -- 记录TCP连接层面相关信息 -- 记录TCP包层面相关信息 -- 从TCP包中提取HTTP1/1.1相关信息 -- 暴露HTTP接口提供给Prometheus以进行可视化 - -项目开发规划： - -- [x] 搭建基础开发框架和自动编译管道。 -- [x] 根据相关工具代码，设计并实现TCP连接信息的记录 -- [x] 根据相关工具代码，设计并实现各个TCP连接发送与接收包信息的记录 -- [ ] 更具相关工具代码，设计并实现TCP错误包信息的记录 -- [ ] 调查相关资料，实现从TCP包中抽取HTTP信息并记录 -- [ ] 讨论与Prometheus的连接方案并实现 -- [ ] 设计并添加工具的控制参数，提高工具的可用性 -- [ ] 编写相关文档 \ No newline at end of file + -a, --all set to trace CLOSED connection + -d, --dport=DPORT trace this destination port only + -s, --sport=SPORT trace this source port only + -?, --help Give this help list + --usage Give a short usage message +``` diff --git a/eBPF_Supermarket/Network_Subsystem/tcp_watch/data/connects.log b/eBPF_Supermarket/Network_Subsystem/tcp_watch/data/connects.log index e69de29bb..82e0ff019 100644 --- a/eBPF_Supermarket/Network_Subsystem/tcp_watch/data/connects.log +++ b/eBPF_Supermarket/Network_Subsystem/tcp_watch/data/connects.log @@ -0,0 +1 @@ +connection{sock="0xffff9ddca18cd780",src="10.0.2.15:56680",dst="220.181.38.149:80",backlog="0",maxbacklog="0",cwnd="10",ssthresh="2147483647",sndbuf="87040",wmem_queued="1280",rx="2.781K",tx="78",srtt="44112",duration="14882"} 0 diff --git a/eBPF_Supermarket/Network_Subsystem/tcp_watch/data/packets.log b/eBPF_Supermarket/Network_Subsystem/tcp_watch/data/packets.log index 7d48ffd47..eae7c0ef3 100644 --- a/eBPF_Supermarket/Network_Subsystem/tcp_watch/data/packets.log +++ b/eBPF_Supermarket/Network_Subsystem/tcp_watch/data/packets.log @@ -1 +1,3 @@ -packet{sock="",src="",dst="",ack="",seq="",mac_delay="",ip_delay="",tcp_delay=""} 0 \ No newline at end of file +packet{sock="0xffff9ddca18cd780",comm="curl",seq="2779274508",ack="504640002",mac_time="1",ip_time="3",tcp_time="14",rx="0"} 0 +packet{sock="0xffff9ddca18cd780",comm="curl",seq="504640002",ack="2779274585",mac_time="45",ip_time="8",tcp_time="489",rx="1"} 0 +packet{sock="0xffff9ddca18cd780",comm="curl",seq="504642722",ack="2779274585",mac_time="37",ip_time="4",tcp_time="194",rx="1"} 0 diff --git a/eBPF_Supermarket/Network_Subsystem/tcp_watch/tcpwatch.bpf.c b/eBPF_Supermarket/Network_Subsystem/tcp_watch/tcpwatch.bpf.c index eddcc7796..7cc2df020 100644 --- a/eBPF_Supermarket/Network_Subsystem/tcp_watch/tcpwatch.bpf.c +++ b/eBPF_Supermarket/Network_Subsystem/tcp_watch/tcpwatch.bpf.c @@ -84,20 +84,86 @@ struct { const volatile int filter_dport = 0; const volatile int filter_sport = 0; +const volatile int all_conn = 0; /* help macro */ #define FILTER_DPORT \ if (filter_dport) { \ - if (dport != filter_dport) { \ + if (conn.dport != filter_dport) { \ return 0; \ } \ } #define FILTER_SPORT \ if (filter_sport) { \ - if (sport != filter_sport) { \ + if (conn.sport != filter_sport) { \ return 0; \ } \ } + +#define CONN_INIT \ + struct conn_t conn = {}; \ + u16 protocol = BPF_CORE_READ(sk, sk_protocol); \ + if (protocol != IPPROTO_TCP) \ + return 0; \ + bpf_get_current_comm(&conn.comm, sizeof(conn.comm)); \ + conn.sock = sk; \ + u16 family = BPF_CORE_READ(sk, __sk_common.skc_family); \ + __be16 dport = BPF_CORE_READ(sk, __sk_common.skc_dport); \ + u16 sport = BPF_CORE_READ(sk, __sk_common.skc_num); \ + conn.family = family; \ + conn.sport = sport; \ + conn.dport = __bpf_ntohs(dport); \ + conn.init_timestamp = bpf_ktime_get_ns() / 1000; \ + conn.ptid = bpf_get_current_pid_tgid(); + +#define CONN_ADD_ADDRESS \ + if (family == AF_INET) { \ + conn.saddr = BPF_CORE_READ(sk, __sk_common.skc_rcv_saddr); \ + conn.daddr = BPF_CORE_READ(sk, __sk_common.skc_daddr); \ + } else if (family == AF_INET6) { \ + bpf_probe_read_kernel( \ + &conn.saddr_v6, \ + sizeof(sk->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32), \ + &sk->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32); \ + bpf_probe_read_kernel( \ + &conn.daddr_v6, \ + sizeof(sk->__sk_common.skc_v6_daddr.in6_u.u6_addr32), \ + &sk->__sk_common.skc_v6_daddr.in6_u.u6_addr32); \ + } + +#define CONN_ADD_EXTRA_INFO \ + struct tcp_sock *tp = (struct tcp_sock *)sk; \ + conn->srtt = BPF_CORE_READ(tp, srtt_us); \ + conn->duration = bpf_ktime_get_ns() / 1000 - conn->init_timestamp; \ + conn->bytes_acked = BPF_CORE_READ(tp, bytes_acked); \ + conn->bytes_received = BPF_CORE_READ(tp, bytes_received); \ + conn->snd_cwnd = BPF_CORE_READ(tp, snd_cwnd); \ + conn->snd_ssthresh = BPF_CORE_READ(tp, snd_ssthresh); \ + conn->sndbuf = BPF_CORE_READ(sk, sk_sndbuf); \ + conn->sk_wmem_queued = BPF_CORE_READ(sk, sk_wmem_queued); \ + conn->tcp_backlog = BPF_CORE_READ(sk, sk_ack_backlog); \ + conn->max_tcp_backlog = BPF_CORE_READ(sk, sk_max_ack_backlog); + +#define CONN_INFO_TRANSFER \ + tinfo->sk = conn->sock; \ + for (int i = 0; i < MAX_COMM; ++i) { \ + tinfo->comm[i] = conn->comm[i]; \ + } + +#define PACKET_INIT_WITH_COMMON_INFO \ + struct pack_t *packet; \ + packet = bpf_ringbuf_reserve(&rb, sizeof(*packet), 0); \ + if (!packet) { \ + return 0; \ + } \ + for (int i = 0; i < MAX_COMM; ++i) { \ + packet->comm[i] = tinfo->comm[i]; \ + } \ + packet->err = 0; \ + packet->sock = sk; \ + packet->ack = pkt_tuple.ack; \ + packet->seq = pkt_tuple.seq; + /* help macro end */ /* help functions */ @@ -156,48 +222,22 @@ static void get_pkt_tuple_v6(struct packet_tuple *pkt_tuple, accecpt an TCP connection */ SEC("kretprobe/inet_csk_accept") -int BPF_KRETPROBE(inet_csk_accept_exit, struct sock *newsk) { +int BPF_KRETPROBE(inet_csk_accept_exit, + struct sock *sk) { // this func return a newsk // bpf_printk("inet_accept_ret\n"); - if (newsk == NULL) { + if (sk == NULL) { // bpf_printk("inet_accept_ret err: newsk is null\n"); return 0; } - u16 protocol = BPF_CORE_READ(newsk, sk_protocol); - if (protocol != IPPROTO_TCP) - return 0; - struct conn_t conn = {}; - bpf_get_current_comm(&conn.comm, sizeof(conn.comm)); - conn.sock = newsk; - - u16 family = BPF_CORE_READ(newsk, __sk_common.skc_family); - __be16 dport = BPF_CORE_READ(newsk, __sk_common.skc_dport); - u16 sport = BPF_CORE_READ(newsk, __sk_common.skc_num); + CONN_INIT - conn.family = family; - conn.sport = sport; - conn.dport = __bpf_ntohs(dport); - - // ##FILTER_PORT## - - if (family == AF_INET) { - conn.saddr = BPF_CORE_READ(newsk, __sk_common.skc_rcv_saddr); - conn.daddr = BPF_CORE_READ(newsk, __sk_common.skc_daddr); - } else if (family == AF_INET6) { - bpf_probe_read_kernel( - &conn.saddr_v6, - sizeof(newsk->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32), - &newsk->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32); - bpf_probe_read_kernel( - &conn.daddr_v6, - sizeof(newsk->__sk_common.skc_v6_daddr.in6_u.u6_addr32), - &newsk->__sk_common.skc_v6_daddr.in6_u.u6_addr32); - } + FILTER_DPORT + FILTER_SPORT - conn.init_timestamp = bpf_ktime_get_ns() / 1000; - conn.ptid = bpf_get_current_pid_tgid(); + CONN_ADD_ADDRESS - int err = bpf_map_update_elem(&conns_info, &newsk, &conn, BPF_ANY); + int err = bpf_map_update_elem(&conns_info, &sk, &conn, BPF_ANY); if (err) { // bpf_printk("inet_accept update err.\n"); return 0; @@ -215,8 +255,8 @@ int BPF_KRETPROBE(inet_csk_accept_exit, struct sock *newsk) { SEC("kprobe/tcp_v4_connect") int BPF_KPROBE(tcp_v4_connect, const struct sock *sk) { // bpf_printk("tcp_v4_connect\n"); - u32 pid = bpf_get_current_pid_tgid(); - int err = bpf_map_update_elem(&sock_stores, &pid, &sk, BPF_ANY); + u32 ptid = bpf_get_current_pid_tgid(); + int err = bpf_map_update_elem(&sock_stores, &ptid, &sk, BPF_ANY); if (err) { // bpf_printk("tcp_v4_connect update sock_stores err.\n"); return 0; @@ -226,36 +266,24 @@ int BPF_KPROBE(tcp_v4_connect, const struct sock *sk) { SEC("kretprobe/tcp_v4_connect") int BPF_KRETPROBE(tcp_v4_connect_exit, int ret) { - u32 pid = bpf_get_current_pid_tgid(); - struct sock **skp = bpf_map_lookup_elem(&sock_stores, &pid); + u32 ptid = bpf_get_current_pid_tgid(); + struct sock **skp = bpf_map_lookup_elem(&sock_stores, &ptid); if (skp == NULL) { return 0; } // bpf_printk("tcp_v4_connect_exit\n"); if (ret != 0) { // bpf_printk("tcp_v4_connect_exit but ret %d\n", ret); - bpf_map_delete_elem(&sock_stores, &pid); + bpf_map_delete_elem(&sock_stores, &ptid); return 0; } struct sock *sk = *skp; - struct conn_t conn = {}; - bpf_get_current_comm(&conn.comm, sizeof(conn.comm)); - conn.sock = sk; - conn.ptid = pid; - u16 family = BPF_CORE_READ(sk, __sk_common.skc_family); - __be16 dport = BPF_CORE_READ(sk, __sk_common.skc_dport); - u16 sport = BPF_CORE_READ(sk, __sk_common.skc_num); + CONN_INIT - conn.family = family; - conn.sport = sport; - conn.dport = __bpf_ntohs(dport); + FILTER_DPORT + FILTER_SPORT - // ##FILTER_PORT## - - conn.saddr = BPF_CORE_READ(sk, __sk_common.skc_rcv_saddr); - conn.daddr = BPF_CORE_READ(sk, __sk_common.skc_daddr); - - conn.init_timestamp = bpf_ktime_get_ns() / 1000; + CONN_ADD_ADDRESS long err = bpf_map_update_elem(&conns_info, &sk, &conn, BPF_ANY); if (err) { @@ -280,41 +308,25 @@ int BPF_KPROBE(tcp_v6_connect, const struct sock *sk) { SEC("kretprobe/tcp_v6_connect") int BPF_KRETPROBE(tcp_v6_connect_exit, int ret) { - u32 pid = bpf_get_current_pid_tgid(); - struct sock **skp = bpf_map_lookup_elem(&sock_stores, &pid); + u32 ptid = bpf_get_current_pid_tgid(); + struct sock **skp = bpf_map_lookup_elem(&sock_stores, &ptid); if (skp == NULL) { return 0; } // bpf_printk("tcp_v6_connect_exit\n"); if (ret != 0) { // bpf_printk("tcp_v6_connect_exit but return %d\n", ret); - bpf_map_delete_elem(&sock_stores, &pid); + bpf_map_delete_elem(&sock_stores, &ptid); return 0; } struct sock *sk = *skp; - struct conn_t conn = {}; - bpf_get_current_comm(&conn.comm, sizeof(conn.comm)); - conn.sock = sk; - conn.ptid = pid; - u16 family = BPF_CORE_READ(sk, __sk_common.skc_family); - __be16 dport = BPF_CORE_READ(sk, __sk_common.skc_dport); - u16 sport = BPF_CORE_READ(sk, __sk_common.skc_num); - conn.family = family; - conn.sport = sport; - conn.dport = __bpf_ntohs(dport); + CONN_INIT - // ##FILTER_PORT## + FILTER_DPORT + FILTER_SPORT - bpf_probe_read_kernel( - &conn.saddr_v6, - sizeof(sk->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32), - &sk->__sk_common.skc_v6_rcv_saddr.in6_u.u6_addr32); - bpf_probe_read_kernel(&conn.daddr_v6, - sizeof(sk->__sk_common.skc_v6_daddr.in6_u.u6_addr32), - &sk->__sk_common.skc_v6_daddr.in6_u.u6_addr32); - - conn.init_timestamp = bpf_ktime_get_ns() / 1000; + CONN_ADD_ADDRESS long err = bpf_map_update_elem(&conns_info, &sk, &conn, BPF_ANY); if (err) { @@ -332,6 +344,9 @@ int BPF_KRETPROBE(tcp_v6_connect_exit, int ret) { /* erase CLOSED TCP connection */ SEC("kprobe/tcp_set_state") int BPF_KPROBE(tcp_set_state, struct sock *sk, int state) { + if (all_conn) { + return 0; + } struct conn_t *value = bpf_map_lookup_elem(&conns_info, &sk); if (state == TCP_CLOSE && value != NULL) { // delete @@ -385,9 +400,6 @@ int BPF_KPROBE(eth_type_trans, struct sk_buff *skb) { struct packet_tuple pkt_tuple = {}; get_pkt_tuple(&pkt_tuple, ip, tcp); - // FILTER_DPORT - // FILTER_SPORT - struct ktime_info *tinfo, zero = {}; tinfo = (struct ktime_info *)bpf_map_lookup_or_try_init( @@ -406,9 +418,6 @@ int BPF_KPROBE(eth_type_trans, struct sk_buff *skb) { struct packet_tuple pkt_tuple = {}; get_pkt_tuple_v6(&pkt_tuple, ip6h, tcp); - // FILTER_DPORT - // FILTER_SPORT - struct ktime_info *tinfo, zero = {}; tinfo = (struct ktime_info *)bpf_map_lookup_or_try_init( @@ -434,9 +443,6 @@ int BPF_KPROBE(ip_rcv_core, struct sk_buff *skb) { struct packet_tuple pkt_tuple = {}; get_pkt_tuple(&pkt_tuple, ip, tcp); - // FILTER_DPORT - // FILTER_SPORT - struct ktime_info *tinfo; tinfo = bpf_map_lookup_elem(×tamps, &pkt_tuple); if (tinfo == NULL) { @@ -458,9 +464,6 @@ int BPF_KPROBE(ip6_rcv_core, struct sk_buff *skb) { struct packet_tuple pkt_tuple = {}; get_pkt_tuple_v6(&pkt_tuple, ip6h, tcp); - // FILTER_DPORT - // FILTER_SPORT - struct ktime_info *tinfo; tinfo = bpf_map_lookup_elem(×tamps, &pkt_tuple); if (tinfo == NULL) { @@ -482,9 +485,6 @@ int BPF_KPROBE(tcp_v4_rcv, struct sk_buff *skb) { struct packet_tuple pkt_tuple = {}; get_pkt_tuple(&pkt_tuple, ip, tcp); - // FILTER_DPORT - // FILTER_SPORT - struct ktime_info *tinfo; tinfo = bpf_map_lookup_elem(×tamps, &pkt_tuple); if (tinfo == NULL) { @@ -506,9 +506,6 @@ int BPF_KPROBE(tcp_v6_rcv, struct sk_buff *skb) { struct packet_tuple pkt_tuple = {}; get_pkt_tuple_v6(&pkt_tuple, ip6h, tcp); - // FILTER_DPORT - // FILTER_SPORT - struct ktime_info *tinfo; tinfo = bpf_map_lookup_elem(×tamps, &pkt_tuple); if (tinfo == NULL) { @@ -541,24 +538,13 @@ int BPF_KPROBE(tcp_v4_do_rcv, struct sock *sk, struct sk_buff *skb) { if (tinfo == NULL) { return 0; } - tinfo->sk = sk; - // copy comm string - for (int i = 0; i < MAX_COMM; ++i) { - tinfo->comm[i] = conn->comm[i]; - } + + CONN_INFO_TRANSFER + // bpf_printk("rx enter tcp4_do_rcv, sk: %p \n", sk); - // conn info update - struct tcp_sock *tp = (struct tcp_sock *)sk; - conn->srtt = BPF_CORE_READ(tp, srtt_us); - conn->duration = bpf_ktime_get_ns() / 1000 - conn->init_timestamp; - conn->bytes_acked = BPF_CORE_READ(tp, bytes_acked); - conn->bytes_received = BPF_CORE_READ(tp, bytes_received); - conn->snd_cwnd = BPF_CORE_READ(tp, snd_cwnd); - conn->snd_ssthresh = BPF_CORE_READ(tp, snd_ssthresh); - conn->sndbuf = BPF_CORE_READ(sk, sk_sndbuf); - conn->sk_wmem_queued = BPF_CORE_READ(sk, sk_wmem_queued); - conn->tcp_backlog = BPF_CORE_READ(sk, sk_ack_backlog); - conn->max_tcp_backlog = BPF_CORE_READ(sk, sk_max_ack_backlog); + + CONN_ADD_EXTRA_INFO + return 0; } SEC("kprobe/tcp_v6_do_rcv") @@ -583,24 +569,12 @@ int BPF_KPROBE(tcp_v6_do_rcv, struct sock *sk, struct sk_buff *skb) { if (tinfo == NULL) { return 0; } - tinfo->sk = sk; - for (int i = 0; i < MAX_COMM; ++i) { - tinfo->comm[i] = conn->comm[i]; - } + CONN_INFO_TRANSFER + // bpf_printk("rx enter tcp6_do_rcv, sk: %p \n", sk); - /*----- update conn info ------*/ - struct tcp_sock *tp = (struct tcp_sock *)sk; - conn->srtt = BPF_CORE_READ(tp, srtt_us); - conn->duration = bpf_ktime_get_ns() / 1000 - conn->init_timestamp; - conn->bytes_acked = BPF_CORE_READ(tp, bytes_acked); - conn->bytes_received = BPF_CORE_READ(tp, bytes_received); - conn->snd_cwnd = BPF_CORE_READ(tp, snd_cwnd); - conn->snd_ssthresh = BPF_CORE_READ(tp, snd_ssthresh); - conn->sndbuf = BPF_CORE_READ(sk, sk_sndbuf); - conn->sk_wmem_queued = BPF_CORE_READ(sk, sk_wmem_queued); - conn->tcp_backlog = BPF_CORE_READ(sk, sk_ack_backlog); - conn->max_tcp_backlog = BPF_CORE_READ(sk, sk_max_ack_backlog); + + CONN_ADD_EXTRA_INFO return 0; } @@ -619,9 +593,6 @@ int BPF_KPROBE(skb_copy_datagram_iter, struct sk_buff *skb) { struct iphdr *ip = skb_to_iphdr(skb); get_pkt_tuple(&pkt_tuple, ip, tcp); - // FILTER_DPORT - // FILTER_SPORT - tinfo = bpf_map_lookup_elem(×tamps, &pkt_tuple); if (tinfo == NULL) { return 0; @@ -633,9 +604,6 @@ int BPF_KPROBE(skb_copy_datagram_iter, struct sk_buff *skb) { struct ipv6hdr *ip6h = skb_to_ipv6hdr(skb); get_pkt_tuple_v6(&pkt_tuple, ip6h, tcp); - // FILTER_DPORT - // FILTER_SPORT - if ((tinfo = bpf_map_lookup_elem(×tamps, &pkt_tuple)) == NULL) { return 0; } @@ -653,27 +621,20 @@ int BPF_KPROBE(skb_copy_datagram_iter, struct sk_buff *skb) { return 0; } // bpf_printk("rx enter app layer.\n"); - struct pack_t *packet; - packet = bpf_ringbuf_reserve(&rb, sizeof(*packet), 0); - if (!packet) { - return 0; - } - // bpf_printk("rx packet sk: %p\n", sk); - for (int i = 0; i < MAX_COMM; ++i) { - packet->comm[i] = tinfo->comm[i]; - } - packet->err = 0; - packet->sock = sk; - packet->ack = pkt_tuple.ack; - packet->seq = pkt_tuple.seq; + + PACKET_INIT_WITH_COMMON_INFO + packet->mac_time = tinfo->ip_time - tinfo->mac_time; packet->ip_time = tinfo->tcp_time - tinfo->ip_time; packet->tcp_time = tinfo->app_time - tinfo->tcp_time; packet->rx = 1; + + // RX HTTP INFO int doff = BPF_CORE_READ_BITFIELD_PROBED(tcp, doff); // 得用bitfield_probed unsigned char *user_data = (unsigned char *)((unsigned char *)tcp + (doff * 4)); bpf_probe_read_str(packet->data, sizeof(packet->data), user_data); + bpf_ringbuf_submit(packet, 0); return 0; } @@ -835,31 +796,20 @@ int BPF_KPROBE(tcp_sendmsg, struct sock *sk, struct msghdr *msg, size_t size) { } tinfo->tcp_time = bpf_ktime_get_ns() / 1000; } - for (int i = 0; i < MAX_COMM; ++i) { - tinfo->comm[i] = conn->comm[i]; - } - tinfo->sk = sk; - /*----- update conn info ------*/ - struct tcp_sock *tp = (struct tcp_sock *)sk; - conn->srtt = BPF_CORE_READ(tp, srtt_us); - conn->duration = bpf_ktime_get_ns() / 1000 - conn->init_timestamp; - conn->bytes_acked = BPF_CORE_READ(tp, bytes_acked); - conn->bytes_received = BPF_CORE_READ(tp, bytes_received); - conn->snd_cwnd = BPF_CORE_READ(tp, snd_cwnd); - conn->snd_ssthresh = BPF_CORE_READ(tp, snd_ssthresh); - conn->sndbuf = BPF_CORE_READ(sk, sk_sndbuf); - conn->sk_wmem_queued = BPF_CORE_READ(sk, sk_wmem_queued); - conn->tcp_backlog = BPF_CORE_READ(sk, sk_ack_backlog); - conn->max_tcp_backlog = BPF_CORE_READ(sk, sk_max_ack_backlog); + + CONN_INFO_TRANSFER + + CONN_ADD_EXTRA_INFO + + // TX HTTP info unsigned char *user_data = BPF_CORE_READ(msg, msg_iter.iov, iov_base); tinfo = (struct ktime_info *)bpf_map_lookup_or_try_init(×tamps, &pkt_tuple, &zero); if (tinfo == NULL) { return 0; } - bpf_printk("kernel data: %s", user_data); bpf_probe_read_str(tinfo->data, sizeof(tinfo->data), user_data); - bpf_printk("tinfo data: %s", tinfo->data); + return 0; } @@ -887,9 +837,6 @@ int BPF_KPROBE(ip_queue_xmit, struct sock *sk, struct sk_buff *skb) { pkt_tuple.seq = __bpf_ntohl(seq); pkt_tuple.ack = __bpf_ntohl(ack); - // FILTER_DPORT - // FILTER_SPORT - if ((tinfo = bpf_map_lookup_elem(×tamps, &pkt_tuple)) == NULL) { return 0; } @@ -931,9 +878,6 @@ int BPF_KPROBE(inet6_csk_xmit, struct sock *sk, struct sk_buff *skb) { pkt_tuple.seq = __bpf_ntohl(seq); pkt_tuple.ack = __bpf_ntohl(ack); - // FILTER_DPORT - // FILTER_SPORT - if ((tinfo = bpf_map_lookup_elem(×tamps, &pkt_tuple)) == NULL) { return 0; } @@ -970,9 +914,6 @@ int BPF_KPROBE(__dev_queue_xmit, struct sk_buff *skb) { struct ipv6hdr *ip6h = skb_to_ipv6hdr(skb); get_pkt_tuple_v6(&pkt_tuple, ip6h, tcp); - // FILTER_DPORT - // FILTER_SPORT - if ((tinfo = bpf_map_lookup_elem(×tamps, &pkt_tuple)) == NULL) { return 0; } @@ -997,9 +938,6 @@ int BPF_KPROBE(dev_hard_start_xmit, struct sk_buff *skb) { struct iphdr *ip = skb_to_iphdr(skb); get_pkt_tuple(&pkt_tuple, ip, tcp); - // FILTER_DPORT - // FILTER_SPORT - if ((tinfo = bpf_map_lookup_elem(×tamps, &pkt_tuple)) == NULL) { return 0; } @@ -1009,9 +947,6 @@ int BPF_KPROBE(dev_hard_start_xmit, struct sk_buff *skb) { struct ipv6hdr *ip6h = skb_to_ipv6hdr(skb); get_pkt_tuple_v6(&pkt_tuple, ip6h, tcp); - // FILTER_DPORT - // FILTER_SPORT - if ((tinfo = bpf_map_lookup_elem(×tamps, &pkt_tuple)) == NULL) { return 0; } @@ -1027,25 +962,17 @@ int BPF_KPROBE(dev_hard_start_xmit, struct sk_buff *skb) { if (sk == NULL) { return 0; } - struct pack_t *packet; - packet = bpf_ringbuf_reserve(&rb, sizeof(*packet), 0); - if (!packet) { - return 0; - } - for (int i = 0; i < MAX_COMM; ++i) { - packet->comm[i] = tinfo->comm[i]; - } - // bpf_printk("tx packet sk: %p\n", sk); - packet->err = 0; - packet->sock = sk; - packet->ack = pkt_tuple.ack; - packet->seq = pkt_tuple.seq; + + PACKET_INIT_WITH_COMMON_INFO + packet->tcp_time = tinfo->ip_time - tinfo->tcp_time; packet->ip_time = tinfo->mac_time - tinfo->ip_time; packet->mac_time = tinfo->qdisc_time - tinfo->mac_time; packet->rx = 0; + + // TX HTTP Info bpf_probe_read_str(packet->data, sizeof(packet->data), tinfo->data); - // 此时skb为非线性 不能直接用RX同样的方式读取 + bpf_ringbuf_submit(packet, 0); return 0; diff --git a/eBPF_Supermarket/Network_Subsystem/tcp_watch/tcpwatch.c b/eBPF_Supermarket/Network_Subsystem/tcp_watch/tcpwatch.c index 326c17764..25a4c5ba2 100644 --- a/eBPF_Supermarket/Network_Subsystem/tcp_watch/tcpwatch.c +++ b/eBPF_Supermarket/Network_Subsystem/tcp_watch/tcpwatch.c @@ -33,10 +33,12 @@ static volatile bool exiting = false; static int sport = 0, dport = 0; // for filter +static int all_conn = 0; // flag static const char argp_program_doc[] = "Watch tcp/ip in network subsystem \n"; static const struct argp_option opts[] = { + {"all", 'a', 0, 0, "set to trace CLOSED connection"}, {"sport", 's', "SPORT", 0, "trace this source port only"}, {"dport", 'd', "DPORT", 0, "trace this destination port only"}, {}}; @@ -44,6 +46,9 @@ static const struct argp_option opts[] = { static error_t parse_arg(int key, char *arg, struct argp_state *state) { char *end; switch (key) { + case 'a': + all_conn = 1; + break; case 's': sport = strtoul(arg, &end, 10); break; @@ -141,26 +146,55 @@ static int print_conns(struct tcpwatch_bpf *skel) { } static int print_packet(void *ctx, void *packet_info, size_t size) { + const struct pack_t *pack_info = packet_info; if (pack_info->err) { + FILE *file = fopen("./data/err.log", "a"); + char reason[20]; if (pack_info->err == 1) { printf("[X] invalid SEQ: sock = %p,comm = %s,seq= %u,ack = %u\n", pack_info->sock, pack_info->comm, pack_info->seq, pack_info->ack); + sprintf(reason, "Invalid SEQ"); } else if (pack_info->err == 2) { printf("[X] invalid checksum: sock = %p,comm = %s\n", pack_info->sock, pack_info->comm); + sprintf(reason, "Invalid checksum"); } else { printf("UNEXPECTED packet error %d.\n", pack_info->err); + sprintf(reason, "Unkonwn"); } + fprintf(file, + "error{sock=\"%p\",comm=\"%s\",seq=\"%u\",ack=\"%u\"," + "reason=\"%s\"} 0\n", + pack_info->sock, pack_info->comm, pack_info->seq, + pack_info->ack, reason); + fclose(file); } else { - printf("%-22p %-22s %-10u %-10u %-10llu %-10llu %-10llu %d\n", + FILE *file = fopen("./data/packets.log", "a"); + printf("%-22p %-22s %-10u %-10u %-10llu %-10llu %-10llu %-5d", pack_info->sock, pack_info->comm, pack_info->seq, pack_info->ack, pack_info->mac_time, pack_info->ip_time, pack_info->tcp_time, pack_info->rx); if (strstr((char *)pack_info->data, "HTTP/1")) { - printf("%s\n", pack_info->data); + for (int i = 0; i < sizeof(pack_info->data); ++i) { + if (pack_info->data[i] == '\r') { + break; + } + printf("%c", pack_info->data[i]); + } + } else { + printf("-"); } + printf("\n"); + fprintf(file, + "packet{sock=\"%p\",comm=\"%s\",seq=\"%u\",ack=\"%u\"," + "mac_time=\"%llu\",ip_time=\"%llu\",tcp_time=\"%llu\",rx=\"%" + "d\"} 0\n", + pack_info->sock, pack_info->comm, pack_info->seq, + pack_info->ack, pack_info->mac_time, pack_info->ip_time, + pack_info->tcp_time, pack_info->rx); + fclose(file); } return 0; } @@ -190,9 +224,7 @@ int main(int argc, char **argv) { /* Parameterize BPF code */ skel->rodata->filter_dport = dport; skel->rodata->filter_sport = sport; - fprintf(stdout, "Filter source port: %d\n", skel->rodata->filter_sport); - fprintf(stdout, "Filter destination port: %d\n", - skel->rodata->filter_dport); + skel->rodata->all_conn = all_conn; err = tcpwatch_bpf__load(skel); if (err) { @@ -215,8 +247,22 @@ int main(int argc, char **argv) { goto cleanup; } - printf("%-22s %-22s %-10s %-10s %-10s %-10s %-10s %-5s\n", "SOCK", "COMM", - "SEQ", "ACK", "MAC_TIME", "IP_TIME", "TCP_TIME", "RX"); + printf("%-22s %-22s %-10s %-10s %-10s %-10s %-10s %-5s %s\n", "SOCK", + "COMM", "SEQ", "ACK", "MAC_TIME", "IP_TIME", "TCP_TIME", "RX", + "HTTP"); + FILE *err_file = fopen("./data/err.log", "w"); + if (err_file == NULL) { + fprintf(stderr, "Failed to open err.log: (%s)\n", strerror(errno)); + return 0; + } + fclose(err_file); + FILE *packet_file = fopen("./data/packets.log", "w"); + if (packet_file == NULL) { + fprintf(stderr, "Failed to open packets.log: (%s)\n", strerror(errno)); + return 0; + } + fclose(packet_file); + /* Process events */ while (!exiting) { err = ring_buffer__poll(rb, 100 /* timeout, ms */);