#!/usr/bin/env bash stderr() { echo "$@" 1>&2 } usage() { b=$(basename "$0") echo $b: ERROR: "$@" 1>&2 cat 1>&2 < coverage.out go tool cover -func=coverage.out popd if [ "${branch}" != "HEAD" ]; then git worktree remove --force "$dir" fi } coverage() { case "$1" in -d) shift target="${1?Need to provide a target branch argument}" output_dir="$(mktemp -d)" target_out="${output_dir}/target.txt" head_out="${output_dir}/head.txt" cover "${target}" > "${target_out}" cover "HEAD" > "${head_out}" cat "${target_out}" cat "${head_out}" echo "" target_pct="$(tail -n2 ${target_out} | head -n1 | sed -E 's/.*total.*\t([0-9.]+)%.*/\1/')" head_pct="$(tail -n2 ${head_out} | head -n1 | sed -E 's/.*total.*\t([0-9.]+)%/\1/')" echo "Results: ${target} ${target_pct}% HEAD ${head_pct}%" delta_pct=$(echo "$head_pct - $target_pct" | bc -l) echo "Delta: ${delta_pct}" if [[ $delta_pct = \-* ]]; then echo "Regression!"; target_diff="${output_dir}/target.diff.txt" head_diff="${output_dir}/head.diff.txt" cat "${target_out}" | grep -E '^github.com/pelletier/go-toml' | tr -s "\t " | cut -f 2,3 | sort > "${target_diff}" cat "${head_out}" | grep -E '^github.com/pelletier/go-toml' | tr -s "\t " | cut -f 2,3 | sort > "${head_diff}" diff --side-by-side --suppress-common-lines "${target_diff}" "${head_diff}" return 1 fi return 0 ;; esac cover "${1-HEAD}" } bench() { branch="${1}" out="${2}" replace="${3}" dir="$(mktemp -d)" stderr "Executing benchmark for ${branch} at ${dir}" if [ "${branch}" = "HEAD" ]; then cp -r . "${dir}/" else git worktree add "$dir" "$branch" fi pushd "$dir" if [ "${replace}" != "" ]; then find ./benchmark/ -iname '*.go' -exec sed -i -E "s|github.com/pelletier/go-toml/v2|${replace}|g" {} \; go get "${replace}" fi export GOMAXPROCS=2 nice -n -19 taskset --cpu-list 0,1 go test '-bench=^Benchmark(Un)?[mM]arshal' -count=5 -run=Nothing ./... | tee "${out}" popd if [ "${branch}" != "HEAD" ]; then git worktree remove --force "$dir" fi } fmktemp() { if mktemp --version|grep GNU >/dev/null; then mktemp --suffix=-$1; else mktemp -t $1; fi } benchstathtml() { python3 - $1 <<'EOF' import sys lines = [] stop = False with open(sys.argv[1]) as f: for line in f.readlines(): line = line.strip() if line == "": stop = True if not stop: lines.append(line.split(',')) results = [] for line in reversed(lines[1:]): v2 = float(line[1]) results.append([ line[0].replace("-32", ""), "%.1fx" % (float(line[3])/v2), # v1 "%.1fx" % (float(line[5])/v2), # bs ]) # move geomean to the end results.append(results[0]) del results[0] def printtable(data): print(""" """) for r in data: print(" ".format(*r)) print("""
Benchmarkgo-toml v1BurntSushi/toml
{}{}{}
""") def match(x): return "ReferenceFile" in x[0] or "HugoFrontMatter" in x[0] above = [x for x in results if match(x)] below = [x for x in results if not match(x)] printtable(above) print("
See more") print("""

The table above has the results of the most common use-cases. The table below contains the results of all benchmarks, including unrealistic ones. It is provided for completeness.

""") printtable(below) print('

This table can be generated with ./ci.sh benchmark -a -html.

') print("
") EOF } benchmark() { case "$1" in -d) shift target="${1?Need to provide a target branch argument}" old=`fmktemp ${target}` bench "${target}" "${old}" new=`fmktemp HEAD` bench HEAD "${new}" benchstat "${old}" "${new}" return 0 ;; -a) shift v2stats=`fmktemp go-toml-v2` bench HEAD "${v2stats}" "github.com/pelletier/go-toml/v2" v1stats=`fmktemp go-toml-v1` bench HEAD "${v1stats}" "github.com/pelletier/go-toml" bsstats=`fmktemp bs-toml` bench HEAD "${bsstats}" "github.com/BurntSushi/toml" cp "${v2stats}" go-toml-v2.txt cp "${v1stats}" go-toml-v1.txt cp "${bsstats}" bs-toml.txt if [ "$1" = "-html" ]; then tmpcsv=`fmktemp csv` benchstat -csv -geomean go-toml-v2.txt go-toml-v1.txt bs-toml.txt > $tmpcsv benchstathtml $tmpcsv else benchstat -geomean go-toml-v2.txt go-toml-v1.txt bs-toml.txt fi rm -f go-toml-v2.txt go-toml-v1.txt bs-toml.txt return $? esac bench "${1-HEAD}" `mktemp` } case "$1" in coverage) shift; coverage $@;; benchmark) shift; benchmark $@;; *) usage "bad argument $1";; esac