root@rcmaster:/sys/kernel/debug/tracing# head hwlat_detector/* tracing_thresh ==> hwlat_detector/width <== 500000 ==> hwlat_detector/window <== 1000000 ==> tracing_thresh <== 10 root@rcmaster:/sys/kernel/debug/tracing# echo hwlat > current_tracer; sleep 65; echo 0 > tracing_on; cat trace; echo nop > current_tracer # See https://lwn.net/Articles/641678/ for the meaning of "inner" and "outer". # tracer: hwlat # # entries-in-buffer/entries-written: 64/64 #P:8 # # _-----=> irqs-off # / _----=> need-resched # | / _---=> hardirq/softirq # || / _--=> preempt-depth # ||| / delay # TASK-PID CPU# |||| TIMESTAMP FUNCTION # | | | |||| | | <...>-2512 [000] d... 2584.575347: #1 inner/outer(us): 165/223 ts:1490333353.666940246 nmi-total:9 nmi-count:1 <...>-2512 [001] dn.. 2585.586994: #2 inner/outer(us): 185/200 ts:1490333354.678950385 <...>-2512 [002] dn.. 2586.610779: #3 inner/outer(us): 173/209 ts:1490333355.702960161 <...>-2512 [003] d... 2587.634763: #4 inner/outer(us): 229/210 ts:1490333356.718969641 <...>-2512 [004] d... 2588.658749: #5 inner/outer(us): 181/234 ts:1490333357.750979263 <...>-2512 [005] d... 2589.682887: #6 inner/outer(us): 17/236 ts:1490333358.758988675 <...>-2512 [006] dn.. 2590.706737: #7 inner/outer(us): 211/172 ts:1490333359.798998395 <...>-2512 [007] dn.. 2591.730706: #8 inner/outer(us): 20/238 ts:1490333360.823007948 <...>-2512 [000] dn.. 2592.754690: #9 inner/outer(us): 153/231 ts:1490333361.847017517 <...>-2512 [001] dn.. 2593.778708: #10 inner/outer(us): 187/211 ts:1490333362.871027089 <...>-2512 [002] d... 2594.802784: #11 inner/outer(us): 171/228 ts:1490333363.871036448 <...>-2512 [003] d... 2595.826638: #12 inner/outer(us): 163/188 ts:1490333364.871045814 <...>-2512 [004] d... 2596.850634: #13 inner/outer(us): 170/192 ts:1490333365.923055644 <...>-2512 [005] d... 2597.874641: #14 inner/outer(us): 181/173 ts:1490333366.951065264 <...>-2512 [006] dn.. 2598.898807: #15 inner/outer(us): 173/205 ts:1490333367.991075004 nmi-total:6 nmi-count:1 <...>-2512 [007] d... 2599.922553: #16 inner/outer(us): 171/184 ts:1490333368.999084454 <...>-2512 [000] dn.. 2600.946625: #17 inner/outer(us): 223/170 ts:1490333370.039094195 <...>-2512 [001] dn.. 2601.970782: #18 inner/outer(us): 174/177 ts:1490333370.983103074 <...>-2512 [002] d... 2602.994556: #19 inner/outer(us): 154/206 ts:1490333372.067113224 nmi-total:4 nmi-count:1 <...>-2512 [003] dn.. 2604.018614: #20 inner/outer(us): 208/174 ts:1490333373.111123007 <...>-2512 [004] dn.. 2605.042697: #21 inner/outer(us): 184/215 ts:1490333374.135132619 <...>-2512 [005] d... 2606.066494: #22 inner/outer(us): 175/182 ts:1490333375.143142084 nmi-total:4 nmi-count:1 <...>-2512 [006] dn.. 2607.090517: #23 inner/outer(us): 172/183 ts:1490333376.183151850 <...>-2512 [007] dn.. 2608.114466: #24 inner/outer(us): 174/192 ts:1490333377.207161471 nmi-total:5 nmi-count:1 <...>-2512 [000] dn.. 2609.138450: #25 inner/outer(us): 169/229 ts:1490333378.231171095 <...>-2512 [001] dn.. 2610.162410: #26 inner/outer(us): 210/210 ts:1490333379.255180724 <...>-2512 [002] dn.. 2611.186420: #27 inner/outer(us): 201/187 ts:1490333380.279190354 <...>-2512 [003] dn.. 2612.210393: #28 inner/outer(us): 188/210 ts:1490333381.303199989 <...>-2512 [004] dn.. 2613.234571: #29 inner/outer(us): 223/221 ts:1490333382.327209627 <...>-2512 [005] d... 2614.258531: #30 inner/outer(us): 172/183 ts:1490333383.335219140 <...>-2512 [006] dn.. 2615.282354: #31 inner/outer(us): 209/215 ts:1490333384.375228912 <...>-2512 [007] d... 2616.306292: #32 inner/outer(us): 186/196 ts:1490333385.359238191 <...>-2512 [000] d... 2617.330292: #33 inner/outer(us): 20/223 ts:1490333386.399248001 nmi-total:9 nmi-count:1 <...>-2512 [001] dn.. 2618.354533: #34 inner/outer(us): 209/220 ts:1490333387.447257863 nmi-total:4 nmi-count:1 <...>-2512 [002] d... 2619.378310: #35 inner/outer(us): 184/172 ts:1490333388.471267511 <...>-2512 [003] dn.. 2620.402241: #36 inner/outer(us): 210/204 ts:1490333389.495277181 <...>-2512 [004] d... 2621.426262: #37 inner/outer(us): 183/236 ts:1490333390.495286627 <...>-2512 [005] d... 2622.450286: #38 inner/outer(us): 176/187 ts:1490333391.527296360 <...>-2512 [006] dn.. 2623.474467: #39 inner/outer(us): 215/212 ts:1490333392.563306143 <...>-2512 [007] dn.. 2624.498231: #40 inner/outer(us): 218/230 ts:1490333393.591315854 <...>-2512 [000] d... 2625.522468: #41 inner/outer(us): 169/167 ts:1490333394.595325339 <...>-2512 [001] d... 2626.546220: #42 inner/outer(us): 229/173 ts:1490333395.619335013 <...>-2512 [002] dn.. 2627.570364: #43 inner/outer(us): 177/172 ts:1490333396.663344892 <...>-2512 [003] d... 2628.594148: #44 inner/outer(us): 20/207 ts:1490333397.683354543 nmi-total:4 nmi-count:1 <...>-2512 [004] d... 2629.618153: #45 inner/outer(us): 0/183 ts:1490333398.687364033 nmi-total:11 nmi-count:1 <...>-2512 [005] d... 2630.642158: #46 inner/outer(us): 187/174 ts:1490333399.719373812 nmi-total:4 nmi-count:1 <...>-2512 [006] dn.. 2631.666130: #47 inner/outer(us): 218/235 ts:1490333400.759383653 <...>-2512 [007] dn.. 2632.690106: #48 inner/outer(us): 172/211 ts:1490333401.783393351 nmi-total:5 nmi-count:1 <...>-2512 [000] d... 2633.714086: #49 inner/outer(us): 231/182 ts:1490333402.783402823 nmi-total:4 nmi-count:1 <...>-2512 [001] dn.. 2634.738106: #50 inner/outer(us): 212/172 ts:1490333403.831412756 nmi-total:4 nmi-count:1 <...>-2512 [002] dn.. 2635.762038: #51 inner/outer(us): 197/208 ts:1490333404.855422463 nmi-total:5 nmi-count:1 <...>-2512 [003] d... 2636.786024: #52 inner/outer(us): 163/194 ts:1490333405.843431824 <...>-2512 [004] dn.. 2637.809982: #53 inner/outer(us): 230/221 ts:1490333406.903441887 <...>-2512 [005] d... 2638.834222: #54 inner/outer(us): 21/245 ts:1490333407.911451435 <...>-2512 [006] d... 2639.858007: #55 inner/outer(us): 184/20 ts:1490333408.919461013 <...>-2512 [007] dn.. 2640.881976: #56 inner/outer(us): 226/205 ts:1490333409.975471043 <...>-2512 [000] dn.. 2641.905961: #57 inner/outer(us): 233/226 ts:1490333410.999480768 <...>-2512 [001] dn.. 2642.929943: #58 inner/outer(us): 231/208 ts:1490333412.023490496 <...>-2512 [002] dn.. 2643.953930: #59 inner/outer(us): 172/181 ts:1490333413.047500227 <...>-2512 [003] dn.. 2644.978108: #60 inner/outer(us): 175/193 ts:1490333414.071509958 <...>-2512 [004] dn.. 2646.002056: #61 inner/outer(us): 157/183 ts:1490333415.071519468 <...>-2512 [005] d... 2647.026070: #62 inner/outer(us): 210/227 ts:1490333416.103529261 nmi-total:5 nmi-count:1 <...>-2512 [006] dn.. 2648.050080: #63 inner/outer(us): 210/212 ts:1490333417.143539180 nmi-total:4 nmi-count:1 <...>-2512 [007] d... 2649.073858: #64 inner/outer(us): 171/185 ts:1490333418.119548463