1 #!/bin/bash
2 #
3 # Multiqueue: Using pktgen threads for sending on multiple CPUs
4 # * adding devices to kernel threads which are in the same NUMA node
5 # * bound devices queue's irq affinity to the threads, 1:1 mapping
6 # * notice the naming scheme for keeping device names unique
7 # * nameing scheme: dev@thread_number
8 # * flow variation via random UDP source port
9 #
10 basedir=`dirname $0`
11 source ${basedir}/functions.sh
12 root_check_run_with_sudo "$@"
13 #
14 # Required param: -i dev in $DEV
15 source ${basedir}/parameters.sh
16
17 # Trap EXIT first
18 trap_exit
19
20 # Base Config
21 [ -z "$COUNT" ] && COUNT="20000000" # Zero means indefinitely
22 [ -z "$CLONE_SKB" ] && CLONE_SKB="0"
23
24 # Flow variation random source port between min and max
25 UDP_SRC_MIN=9
26 UDP_SRC_MAX=109
27
28 node=`get_iface_node $DEV`
29 irq_array=(`get_iface_irqs $DEV`)
30 cpu_array=(`get_node_cpus $node`)
31
32 [ $THREADS -gt ${#irq_array[*]} -o $THREADS -gt ${#cpu_array[*]} ] && \
33 err 1 "Thread number $THREADS exceeds: min (${#irq_array[*]},${#cpu_array[*]})"
34
35 # (example of setting default params in your script)
36 if [ -z "$DEST_IP" ]; then
37 [ -z "$IP6" ] && DEST_IP="198.18.0.42" || DEST_IP="FD00::1"
38 fi
39 [ -z "$DST_MAC" ] && DST_MAC="90:e2:ba:ff:ff:ff"
40 if [ -n "$DEST_IP" ]; then
41 validate_addr${IP6} $DEST_IP
42 read -r DST_MIN DST_MAX <<< $(parse_addr${IP6} $DEST_IP)
43 fi
44 if [ -n "$DST_PORT" ]; then
45 read -r UDP_DST_MIN UDP_DST_MAX <<< $(parse_ports $DST_PORT)
46 validate_ports $UDP_DST_MIN $UDP_DST_MAX
47 fi
48
49 # General cleanup everything since last run
50 [ -z "$APPEND" ] && pg_ctrl "reset"
51
52 # Threads are specified with parameter -t value in $THREADS
53 for ((i = 0; i < $THREADS; i++)); do
54 # The device name is extended with @name, using thread number to
55 # make then unique, but any name will do.
56 # Set the queue's irq affinity to this $thread (processor)
57 # if '-f' is designated, offset cpu id
58 thread=${cpu_array[$((i+F_THREAD))]}
59 dev=${DEV}@${thread}
60 echo $thread > /proc/irq/${irq_array[$i]}/smp_affinity_list
61 info "irq ${irq_array[$i]} is set affinity to `cat /proc/irq/${irq_array[$i]}/smp_affinity_list`"
62
63 # Add remove all other devices and add_device $dev to thread
64 [ -z "$APPEND" ] && pg_thread $thread "rem_device_all"
65 pg_thread $thread "add_device" $dev
66
67 # select queue and bind the queue and $dev in 1:1 relationship
68 queue_num=$i
69 info "queue number is $queue_num"
70 pg_set $dev "queue_map_min $queue_num"
71 pg_set $dev "queue_map_max $queue_num"
72
73 # Notice config queue to map to cpu (mirrors smp_processor_id())
74 # It is beneficial to map IRQ /proc/irq/*/smp_affinity 1:1 to CPU number
75 pg_set $dev "flag QUEUE_MAP_CPU"
76
77 # Base config of dev
78 pg_set $dev "count $COUNT"
79 pg_set $dev "clone_skb $CLONE_SKB"
80 pg_set $dev "pkt_size $PKT_SIZE"
81 pg_set $dev "delay $DELAY"
82
83 # Flag example disabling timestamping
84 pg_set $dev "flag NO_TIMESTAMP"
85
86 # Destination
87 pg_set $dev "dst_mac $DST_MAC"
88 pg_set $dev "dst${IP6}_min $DST_MIN"
89 pg_set $dev "dst${IP6}_max $DST_MAX"
90
91 if [ -n "$DST_PORT" ]; then
92 # Single destination port or random port range
93 pg_set $dev "flag UDPDST_RND"
94 pg_set $dev "udp_dst_min $UDP_DST_MIN"
95 pg_set $dev "udp_dst_max $UDP_DST_MAX"
96 fi
97
98 [ ! -z "$UDP_CSUM" ] && pg_set $dev "flag UDPCSUM"
99
100 # Setup random UDP port src range
101 pg_set $dev "flag UDPSRC_RND"
102 pg_set $dev "udp_src_min $UDP_SRC_MIN"
103 pg_set $dev "udp_src_max $UDP_SRC_MAX"
104 done
105
106 # Run if user hits control-c
107 function print_result() {
108 # Print results
109 for ((i = 0; i < $THREADS; i++)); do
110 thread=${cpu_array[$((i+F_THREAD))]}
111 dev=${DEV}@${thread}
112 echo "Device: $dev"
113 cat /proc/net/pktgen/$dev | grep -A2 "Result:"
114 done
115 }
116 # trap keyboard interrupt (Ctrl-C)
117 trap true SIGINT
118
119 # start_run
120 if [ -z "$APPEND" ]; then
121 echo "Running... ctrl^C to stop" >&2
122 pg_ctrl "start"
123 echo "Done" >&2
124
125 print_result
126 else
127 echo "Append mode: config done. Do more or use 'pg_ctrl start' to run"
128 fi
Linux® is a registered trademark of Linus Torvalds in the United States and other countries.
TOMOYO® is a registered trademark of NTT DATA CORPORATION.