#!/bin/sh
|
#
|
# Benchmarking harness for json-c
|
#
|
# Use this to compare the behavior of two different versions of the library,
|
# such as json-c-0.14 release vs master, master vs local changes, etc...
|
#
|
|
set -e
|
|
trap 'echo "FAILED!"' EXIT
|
|
RUNDIR=$(dirname "$0")
|
RUNDIR=$(cd "$RUNDIR" && pwd)
|
|
TOP=$(cd "$RUNDIR/.." && pwd)
|
|
usage()
|
{
|
exitval=$1
|
errmsg=$2
|
if [ $exitval -ne 0 ] ; then
|
exec 1>&2
|
fi
|
if [ ! -z "$errmsg" ] ; then
|
echo "ERROR: $errmsg" 1>&2
|
fi
|
cat <<EOF
|
Usage: $0 [-h] [-v] [--build] [--run] [--compare] ...XAX...
|
EOF
|
|
exit $extival
|
}
|
|
before_arg=
|
after_arg=
|
do_all=1
|
do_build=0
|
do_run=0
|
do_compare=0
|
|
while [ $# -gt 0 ] ; do
|
case "$1" in
|
--before)
|
before_arg=$2
|
shift
|
;;
|
--after)
|
after_arg=$2
|
shift
|
;;
|
--build)
|
do_all=0
|
do_build=1
|
;;
|
--run)
|
do_all=0
|
do_run=1
|
;;
|
--compare)
|
do_all=0
|
do_compare=1
|
;;
|
-h)
|
usage 0 ""
|
;;
|
-v)
|
set -x
|
;;
|
*)
|
usage 1 "Unknown args: $*"
|
;;
|
esac
|
shift
|
done
|
|
WORK="${RUNDIR}/work"
|
mkdir -p "${WORK}"
|
|
DATA="${RUNDIR}/data"
|
mkdir -p "${DATA}"
|
|
for file in citm_catalog.json twitter.json canada.json ; do
|
if [ ! -r "${DATA}/${file}" ] ; then
|
echo "Fetching ${file} from github.com/mloskot/json_benchmark"
|
URL="https://github.com/mloskot/json_benchmark/raw/master/data/${file}"
|
curl -s -L -o "${DATA}/${file}" "$URL"
|
fi
|
done
|
echo
|
|
# Identify "after" commit hash, in order of preference
|
if [ ! -z "$after_arg" -a -d "$after_arg" ] ; then
|
# Use provided directory
|
after_src_dir="$after_arg"
|
after_commit=
|
echo "Using provided directory [$after_arg] as 'after'"
|
else
|
_commit=
|
if [ ! -z "$after_arg" ] ; then
|
# Use provided commit hash
|
_commit=$(git rev-parse --verify "$after_arg")
|
fi
|
if [ ! -z "$_commit" ] ;then
|
after_src_dir= # i.e. current tree
|
after_commit="$_commit"
|
echo "Using provided commit [$after_arg => $_commit] as 'after'"
|
else
|
# Local changes in current working directory
|
# ${cur_branch}
|
after_src_dir=$TOP
|
after_commit=
|
echo "Using local changes in $TOP as 'after'"
|
fi
|
fi
|
|
# Identify "before" commit hash, in order of preference
|
if [ ! -z "$before_arg" -a -d "$before_arg" ] ; then
|
# Use provided directory
|
before_src_dir="$before_arg"
|
before_commit=
|
echo "Using provided directory [$before_arg] as 'before'"
|
else
|
_commit=
|
if [ ! -z "$before_arg" ] ; then
|
# Use provided commit hash
|
_commit=$(git rev-parse --verify "$before_arg")
|
fi
|
if [ ! -z "$_commit" ] ;then
|
before_src_dir= # i.e. current tree
|
before_commit="$_commit"
|
echo "Using provided commit [$before_arg => $_commit] as 'before'"
|
else
|
# Use origin/${cur_branch}, if different from ${after_commit}
|
_cur_branch=$(git rev-parse --abbrev-ref HEAD)
|
_commit=
|
if [ ! -z "${_cur_branch}" ] ; then
|
_commit=$(git rev-parse --verify "origin/${_cur_branch}")
|
echo "Using origin/${_cur_branch} [$_commit] as 'before'"
|
fi
|
if [ "$_commit" = "${after_commit}" ] ; then
|
_commit=
|
fi
|
fi
|
|
if [ ! -z "$_commit" ] ; then
|
before_src_dir= # i.e. current tree
|
before_commit="$_commit"
|
else
|
# Use previous release
|
before_src_dir= # i.e. current tree
|
before_commit="$(git tag | sort | tail -1)"
|
echo "Using previous release [$before_commit] as 'before'"
|
fi
|
fi
|
|
echo
|
|
compile_benchmark()
|
{
|
local bname=$1
|
local src_dir="$2"
|
local src_commit="$3"
|
|
local build_dir="${WORK}/$bname/build"
|
local inst_dir="${WORK}/$bname/install"
|
local bench_dir="${WORK}/$bname/bench"
|
|
echo
|
echo "=========== $bname ==========="
|
echo
|
|
mkdir -p "${build_dir}"
|
mkdir -p "${inst_dir}"
|
mkdir -p "${bench_dir}"
|
|
if [ ! -z "$src_commit" ] ; then
|
# Resolve the short hash, tag or branch name to full hash
|
src_commit=$(git rev-parse $src_commit)
|
fi
|
|
# No src dir specified, clone and checkout $src_commit
|
if [ -z "$src_dir" ] ; then
|
src_dir="${WORK}/$bname/src"
|
echo "=== Using sources in $src_dir"
|
mkdir -p "$src_dir"
|
at_commit=$(git --git-dir="$src_dir/.git" rev-parse HEAD 2> /dev/null || true)
|
echo "at_commit: $at_commit"
|
if [ -z "$at_commit" ] ; then
|
# Assume it's an empty dir
|
git clone -n "$TOP" "$src_dir"
|
fi
|
git -C "$src_dir" --git-dir="$src_dir/.git" checkout "$src_commit"
|
fi
|
# else, use the provided $src_dir
|
|
if [ -e "${src_dir}/CMakeLists.txt" ] ; then
|
cd "${build_dir}"
|
cmake -DCMAKE_INSTALL_PREFIX="${inst_dir}" "${src_dir}"
|
else
|
# Old versions of json-c used automake/autoconf
|
cd "${src_dir}"
|
sh autogen.sh # always run it, configure doesn't always work
|
cd "${build_dir}"
|
"${src_dir}/configure" --prefix="${inst_dir}"
|
fi
|
make all install
|
|
cd "${bench_dir}"
|
cmake -DCMAKE_PREFIX_PATH="${inst_dir}" "${TOP}/apps"
|
make all
|
}
|
|
# XXX TODO: name "after" and "before" uniquely using the dir & commit
|
|
if [ $do_all -ne 0 -o $do_build -ne 0 ] ; then
|
sleep 5 # Wait slightly, to allow the human to read the message
|
# about what exactly we're doing to benchmark.
|
compile_benchmark "after" "${after_src_dir}" "${after_commit}"
|
compile_benchmark "before" "${before_src_dir}" "${before_commit}"
|
fi
|
|
run_benchmark()
|
{
|
local bname=$1
|
local inst_dir="${WORK}/$bname/install"
|
local bench_dir="${WORK}/$bname/bench"
|
|
local INPUT=${DATA}/canada.json
|
|
cd "${bench_dir}"
|
mkdir -p results
|
(time ./json_parse -n "${INPUT}") > results/basic_timing.out 2>&1
|
valgrind --tool=massif --massif-out-file=massif.out ./json_parse -n "${INPUT}"
|
ms_print massif.out > results/ms_print.out
|
heaptrack -o heaptrack_out ./json_parse -n "${INPUT}"
|
heaptrack --analyze heaptrack_out.gz -H histogram.out > results/heaptrack.out
|
awk ' { s=$1; count=$2; ru=(int((s+ 15) / 16)) * 16; wasted = ((ru-s)*count); print s, count, ru-s, wasted; total=total+wasted} END { print "Total: ", total }' histogram.out > results/histogram2.out
|
|
# XXX stamp some info about what was built & run into ./results/.
|
|
echo "DONE with $bname"
|
}
|
|
if [ $do_all -ne 0 -o $do_run -ne 0 ] ; then
|
run_benchmark "after"
|
run_benchmark "before"
|
fi
|
|
if [ $do_compare -ne 0 ] ; then
|
# XXX this needs better analysis
|
cd "${WORK}"
|
diff -udr before/bench/results after/bench/results || true
|
else
|
echo "To compare results, run:"
|
echo "$0 --compare"
|
fi
|
|
trap '' EXIT
|
|
:<<=cut
|
|
Benchmarks to run:
|
|
* Parse JSON strings, of various sizes and characteristics
|
* Flags: STRICT vs. non-STRICT, validate UTF8
|
|
* Serialization time
|
* plain, spaces, pretty
|
|
* json_c_visit tests
|
* JSON pointer tests
|
|
Things to record and compare:
|
|
* Running time
|
* Peak memory usage
|
* Useful bytes vs. overhead for memory allocations
|
* Total number of allocations
|
* Average allocation size
|
* Log of all allocation sizes
|
|
=cut
|
