Skip to content

Commit

Permalink
add function for merging benchmark result stats
Browse files Browse the repository at this point in the history
  • Loading branch information
Ondřej Benkovský committed Apr 24, 2024
1 parent 54377ca commit 74d90c0
Show file tree
Hide file tree
Showing 4 changed files with 147 additions and 132 deletions.
20 changes: 10 additions & 10 deletions pkg/reporter/jsonreporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ func (s *jsonReporter) print(params reportParameters) error {
var res []histogramPoint

if params.benchmark.HistDisplay {
dist := params.timings.Distribution()
dist := params.hist.Distribution()
for _, d := range dist {
res = append(res, histogramPoint{
LatencyMs: time.Duration(d.To/2 + d.From/2).Milliseconds(),
Expand Down Expand Up @@ -94,15 +94,15 @@ func (s *jsonReporter) print(params reportParameters) error {
ResponseRcodes: codeTotalsMapped,
QuestionTypes: params.qtypeTotals,
LatencyStats: latencyStats{
MinMs: time.Duration(params.timings.Min()).Milliseconds(),
MeanMs: time.Duration(params.timings.Mean()).Milliseconds(),
StdMs: time.Duration(params.timings.StdDev()).Milliseconds(),
MaxMs: time.Duration(params.timings.Max()).Milliseconds(),
P99Ms: time.Duration(params.timings.ValueAtQuantile(99)).Milliseconds(),
P95Ms: time.Duration(params.timings.ValueAtQuantile(95)).Milliseconds(),
P90Ms: time.Duration(params.timings.ValueAtQuantile(90)).Milliseconds(),
P75Ms: time.Duration(params.timings.ValueAtQuantile(75)).Milliseconds(),
P50Ms: time.Duration(params.timings.ValueAtQuantile(50)).Milliseconds(),
MinMs: time.Duration(params.hist.Min()).Milliseconds(),
MeanMs: time.Duration(params.hist.Mean()).Milliseconds(),
StdMs: time.Duration(params.hist.StdDev()).Milliseconds(),
MaxMs: time.Duration(params.hist.Max()).Milliseconds(),
P99Ms: time.Duration(params.hist.ValueAtQuantile(99)).Milliseconds(),
P95Ms: time.Duration(params.hist.ValueAtQuantile(95)).Milliseconds(),
P90Ms: time.Duration(params.hist.ValueAtQuantile(90)).Milliseconds(),
P75Ms: time.Duration(params.hist.ValueAtQuantile(75)).Milliseconds(),
P50Ms: time.Duration(params.hist.ValueAtQuantile(50)).Milliseconds(),
},
LatencyDistribution: res,
DohHTTPResponseStatusCodes: params.dohResponseStatusesTotals,
Expand Down
109 changes: 109 additions & 0 deletions pkg/reporter/merge.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
package reporter

import (
"errors"

Check failure on line 4 in pkg/reporter/merge.go

View workflow job for this annotation

GitHub Actions / golangci

File is not `gofumpt`-ed (gofumpt)
"github.com/HdrHistogram/hdrhistogram-go"

Check failure on line 5 in pkg/reporter/merge.go

View workflow job for this annotation

GitHub Actions / golangci

File is not `gci`-ed with --skip-generated -s standard -s default (gci)
"github.com/tantalor93/dnspyre/v3/pkg/dnsbench"
"net"

Check failure on line 7 in pkg/reporter/merge.go

View workflow job for this annotation

GitHub Actions / golangci

File is not `gofumpt`-ed (gofumpt)
"sort"

Check failure on line 8 in pkg/reporter/merge.go

View workflow job for this annotation

GitHub Actions / golangci

File is not `gci`-ed with --skip-generated -s standard -s default (gci)
)

type BenchmarkResultStats struct {

Check warning on line 11 in pkg/reporter/merge.go

View workflow job for this annotation

GitHub Actions / golangci

exported: exported type BenchmarkResultStats should have comment or be unexported (revive)
Codes map[int]int64
Qtypes map[string]int64
Hist *hdrhistogram.Histogram
Timings []dnsbench.Datapoint
Counters dnsbench.Counters
Errors []dnsbench.ErrorDatapoint
GroupedErrors map[string]int
AuthenticatedDomains map[string]struct{}
DoHStatusCodes map[int]int64
}

func Merge(b *dnsbench.Benchmark, stats []*dnsbench.ResultStats) BenchmarkResultStats {

Check warning on line 23 in pkg/reporter/merge.go

View workflow job for this annotation

GitHub Actions / golangci

exported: exported function Merge should have comment or be unexported (revive)
totals := BenchmarkResultStats{
Codes: make(map[int]int64),
Qtypes: make(map[string]int64),
Hist: hdrhistogram.New(b.HistMin.Nanoseconds(), b.HistMax.Nanoseconds(), b.HistPre),
GroupedErrors: make(map[string]int),
AuthenticatedDomains: make(map[string]struct{}),
DoHStatusCodes: make(map[int]int64),
}

for _, s := range stats {
for _, err := range s.Errors {
errorString := errString(err)

if v, ok := totals.GroupedErrors[errorString]; ok {
totals.GroupedErrors[errorString] = v + 1
} else {
totals.GroupedErrors[errorString] = 1
}
}
totals.Errors = append(totals.Errors, s.Errors...)

totals.Hist.Merge(s.Hist)
totals.Timings = append(totals.Timings, s.Timings...)
if s.Codes != nil {
for k, v := range s.Codes {
totals.Codes[k] += v
}
}
if s.Qtypes != nil {
for k, v := range s.Qtypes {
totals.Qtypes[k] += v
}
}
if s.DoHStatusCodes != nil {
for k, v := range s.DoHStatusCodes {
totals.DoHStatusCodes[k] += v
}
}
if s.Counters != nil {
totals.Counters = dnsbench.Counters{
Total: totals.Counters.Total + s.Counters.Total,
IOError: totals.Counters.IOError + s.Counters.IOError,
Success: totals.Counters.Success + s.Counters.Success,
Negative: totals.Counters.Negative + s.Counters.Negative,
Error: totals.Counters.Error + s.Counters.Error,
IDmismatch: totals.Counters.IDmismatch + s.Counters.IDmismatch,
Truncated: totals.Counters.Truncated + s.Counters.Truncated,
}
}
if b.DNSSEC {
for k := range s.AuthenticatedDomains {
totals.AuthenticatedDomains[k] = struct{}{}
}
}
}

// sort data points from the oldest to the earliest, so we can better plot time dependant graphs (like line)
sort.SliceStable(totals.Timings, func(i, j int) bool {
return totals.Timings[i].Start.Before(totals.Timings[j].Start)
})

// sort error data points from the oldest to the earliest, so we can better plot time dependant graphs (like line)
sort.SliceStable(totals.Errors, func(i, j int) bool {
return totals.Errors[i].Start.Before(totals.Errors[j].Start)
})
return totals
}

func errString(err dnsbench.ErrorDatapoint) string {
var errorString string
var netOpErr *net.OpError
var resolveErr *net.DNSError

switch {
case errors.As(err.Err, &resolveErr):
errorString = resolveErr.Err + " " + resolveErr.Name
case errors.As(err.Err, &netOpErr):
errorString = netOpErr.Op + " " + netOpErr.Net
if netOpErr.Addr != nil {
errorString += " " + netOpErr.Addr.String()
}
default:
errorString = err.Err.Error()
}
return errorString
}
126 changes: 16 additions & 110 deletions pkg/reporter/report.go
Original file line number Diff line number Diff line change
@@ -1,12 +1,9 @@
package reporter

import (
"errors"
"fmt"
"io"
"net"
"os"
"sort"
"time"

"github.com/HdrHistogram/hdrhistogram-go"
Expand All @@ -21,7 +18,7 @@ type orderedMap struct {
type reportParameters struct {
benchmark *dnsbench.Benchmark
outputWriter io.Writer
timings *hdrhistogram.Histogram
hist *hdrhistogram.Histogram
codeTotals map[int]int64
totalCounters dnsbench.Counters
qtypeTotals map[string]int64
Expand All @@ -34,68 +31,15 @@ type reportParameters struct {
// PrintReport prints formatted benchmark result to stdout, exports graphs and generates CSV output if configured.
// If there is a fatal error while printing report, an error is returned.
func PrintReport(b *dnsbench.Benchmark, stats []*dnsbench.ResultStats, benchStart time.Time, benchDuration time.Duration) error {
// merge all the stats here
timings := hdrhistogram.New(b.HistMin.Nanoseconds(), b.HistMax.Nanoseconds(), b.HistPre)
codeTotals := make(map[int]int64)
qtypeTotals := make(map[string]int64)
dohResponseStatusesTotals := make(map[int]int64)
totals := Merge(b, stats)

times := make([]dnsbench.Datapoint, 0)
errTimes := make([]dnsbench.ErrorDatapoint, 0)

errs := make(map[string]int, 0)
top3errs := make(map[string]int)
top3errorsInOrder := make([]string, 0)

var totalCounters dnsbench.Counters

// TODO proper coverage of plots
for _, s := range stats {
for _, err := range s.Errors {
errorString := errString(err)

if v, ok := errs[errorString]; ok {
errs[errorString] = v + 1
} else {
errs[errorString] = 1
}
}
errTimes = append(errTimes, s.Errors...)

timings.Merge(s.Hist)
times = append(times, s.Timings...)
if s.Codes != nil {
for k, v := range s.Codes {
codeTotals[k] += v
}
}
if s.Qtypes != nil {
for k, v := range s.Qtypes {
qtypeTotals[k] += v
}
}
if s.DoHStatusCodes != nil {
for k, v := range s.DoHStatusCodes {
dohResponseStatusesTotals[k] += v
}
}
if s.Counters != nil {
totalCounters = dnsbench.Counters{
Total: totalCounters.Total + s.Counters.Total,
IOError: totalCounters.IOError + s.Counters.IOError,
Success: totalCounters.Success + s.Counters.Success,
Negative: totalCounters.Negative + s.Counters.Negative,
Error: totalCounters.Error + s.Counters.Error,
IDmismatch: totalCounters.IDmismatch + s.Counters.IDmismatch,
Truncated: totalCounters.Truncated + s.Counters.Truncated,
}
}
}

for i := 0; i < 3; i++ {
max := 0
maxerr := ""
for k, v := range errs {
for k, v := range totals.GroupedErrors {
if _, ok := top3errs[k]; v > max && !ok {
maxerr = k
max = v
Expand All @@ -107,28 +51,18 @@ func PrintReport(b *dnsbench.Benchmark, stats []*dnsbench.ResultStats, benchStar
}
}

// sort data points from the oldest to the earliest, so we can better plot time dependant graphs (like line)
sort.SliceStable(times, func(i, j int) bool {
return times[i].Start.Before(times[j].Start)
})

// sort error data points from the oldest to the earliest, so we can better plot time dependant graphs (like line)
sort.SliceStable(errTimes, func(i, j int) bool {
return errTimes[i].Start.Before(errTimes[j].Start)
})

if len(b.PlotDir) != 0 {
now := time.Now().Format(time.RFC3339)
dir := fmt.Sprintf("%s/graphs-%s", b.PlotDir, now)
if err := os.Mkdir(dir, os.ModePerm); err != nil {
panic(err)
}
plotHistogramLatency(fileName(b, dir, "latency-histogram"), times)
plotBoxPlotLatency(fileName(b, dir, "latency-boxplot"), b.Server, times)
plotResponses(fileName(b, dir, "responses-barchart"), codeTotals)
plotLineThroughput(fileName(b, dir, "throughput-lineplot"), benchStart, times)
plotLineLatencies(fileName(b, dir, "latency-lineplot"), benchStart, times)
plotErrorRate(fileName(b, dir, "errorrate-lineplot"), benchStart, errTimes)
plotHistogramLatency(fileName(b, dir, "latency-histogram"), totals.Timings)
plotBoxPlotLatency(fileName(b, dir, "latency-boxplot"), b.Server, totals.Timings)
plotResponses(fileName(b, dir, "responses-barchart"), totals.Codes)
plotLineThroughput(fileName(b, dir, "throughput-lineplot"), benchStart, totals.Timings)
plotLineLatencies(fileName(b, dir, "latency-lineplot"), benchStart, totals.Timings)
plotErrorRate(fileName(b, dir, "errorrate-lineplot"), benchStart, totals.Errors)
}

var csv *os.File
Expand All @@ -148,16 +82,7 @@ func PrintReport(b *dnsbench.Benchmark, stats []*dnsbench.ResultStats, benchStar
}()

if csv != nil {
writeBars(csv, timings.Distribution())
}

authenticatedDomains := make(map[string]struct{})
if b.DNSSEC {
for _, v := range stats {
for k := range v.AuthenticatedDomains {
authenticatedDomains[k] = struct{}{}
}
}
writeBars(csv, totals.Hist.Distribution())
}

if b.Silent {
Expand All @@ -167,14 +92,14 @@ func PrintReport(b *dnsbench.Benchmark, stats []*dnsbench.ResultStats, benchStar
params := reportParameters{
benchmark: b,
outputWriter: b.Writer,
timings: timings,
codeTotals: codeTotals,
totalCounters: totalCounters,
qtypeTotals: qtypeTotals,
hist: totals.Hist,
codeTotals: totals.Codes,
totalCounters: totals.Counters,
qtypeTotals: totals.Qtypes,
topErrs: topErrs,
authenticatedDomains: authenticatedDomains,
authenticatedDomains: totals.AuthenticatedDomains,
benchmarkDuration: benchDuration,
dohResponseStatusesTotals: dohResponseStatusesTotals,
dohResponseStatusesTotals: totals.DoHStatusCodes,
}
if b.JSON {
j := jsonReporter{}
Expand All @@ -184,25 +109,6 @@ func PrintReport(b *dnsbench.Benchmark, stats []*dnsbench.ResultStats, benchStar
return s.print(params)
}

func errString(err dnsbench.ErrorDatapoint) string {
var errorString string
var netOpErr *net.OpError
var resolveErr *net.DNSError

switch {
case errors.As(err.Err, &resolveErr):
errorString = resolveErr.Err + " " + resolveErr.Name
case errors.As(err.Err, &netOpErr):
errorString = netOpErr.Op + " " + netOpErr.Net
if netOpErr.Addr != nil {
errorString += " " + netOpErr.Addr.String()
}
default:
errorString = err.Err.Error()
}
return errorString
}

func fileName(b *dnsbench.Benchmark, dir, name string) string {
return dir + "/" + name + "." + b.PlotFormat
}
Expand Down
24 changes: 12 additions & 12 deletions pkg/reporter/stdreporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -70,17 +70,17 @@ func (s *standardReporter) print(params reportParameters) error {
fmt.Fprintln(params.outputWriter, "Time taken for tests:\t", printutils.HighlightStr(roundDuration(params.benchmarkDuration).String()))
fmt.Fprintf(params.outputWriter, "Questions per second:\t %s", printutils.HighlightStr(fmt.Sprintf("%0.1f", float64(params.totalCounters.Total)/params.benchmarkDuration.Seconds())))
fmt.Fprintln(params.outputWriter)
min := time.Duration(params.timings.Min())
mean := time.Duration(params.timings.Mean())
sd := time.Duration(params.timings.StdDev())
max := time.Duration(params.timings.Max())
p99 := time.Duration(params.timings.ValueAtQuantile(99))
p95 := time.Duration(params.timings.ValueAtQuantile(95))
p90 := time.Duration(params.timings.ValueAtQuantile(90))
p75 := time.Duration(params.timings.ValueAtQuantile(75))
p50 := time.Duration(params.timings.ValueAtQuantile(50))

if tc := params.timings.TotalCount(); tc > 0 {
min := time.Duration(params.hist.Min())
mean := time.Duration(params.hist.Mean())
sd := time.Duration(params.hist.StdDev())
max := time.Duration(params.hist.Max())
p99 := time.Duration(params.hist.ValueAtQuantile(99))
p95 := time.Duration(params.hist.ValueAtQuantile(95))
p90 := time.Duration(params.hist.ValueAtQuantile(90))
p75 := time.Duration(params.hist.ValueAtQuantile(75))
p50 := time.Duration(params.hist.ValueAtQuantile(50))

if tc := params.hist.TotalCount(); tc > 0 {
fmt.Fprintln(params.outputWriter, "DNS timings,", printutils.HighlightStr(tc), "datapoints")
fmt.Fprintln(params.outputWriter, "\t min:\t\t", printutils.HighlightStr(roundDuration(min)))
fmt.Fprintln(params.outputWriter, "\t mean:\t\t", printutils.HighlightStr(roundDuration(mean)))
Expand All @@ -92,7 +92,7 @@ func (s *standardReporter) print(params reportParameters) error {
fmt.Fprintln(params.outputWriter, "\t p75:\t\t", printutils.HighlightStr(roundDuration(p75)))
fmt.Fprintln(params.outputWriter, "\t p50:\t\t", printutils.HighlightStr(roundDuration(p50)))

dist := params.timings.Distribution()
dist := params.hist.Distribution()
if params.benchmark.HistDisplay && tc > 1 {
fmt.Fprintln(params.outputWriter)
fmt.Fprintln(params.outputWriter, "DNS distribution,", printutils.HighlightStr(tc), "datapoints")
Expand Down

0 comments on commit 74d90c0

Please sign in to comment.