Skip to content

Commit 25b8a3c

Browse files
haampiehaampie
authored
Linear time list operations (#10)
Avoids string concatenation overhead in extend and preextend "tricks" used: * Build a local temporary string instead of extending a big string one item at a time (quadratic complexity) * When prefix not set: use `set --` and `"$*"` with different `IFS` for fast concatenation, presumably linear complexity. * When prefix is set: shift the first element, followed by a tight loop (still quadratic complexity). * Eliminate the array reversal in preextend Signed-off-by: Harmen Stoppels <me@harmenstoppels.nl>
1 parent 1e20160 commit 25b8a3c

5 files changed

Lines changed: 409 additions & 46 deletions

File tree

‎.github/workflows/shellcheck.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,6 @@ jobs:
1111
steps:
1212
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
1313
- name: Run shellcheck
14-
run: shellcheck cc.sh test/run.sh
14+
run: shellcheck cc.sh test/*.sh benchmark/*.sh
1515
- name: Run tests
1616
run: sh test/run.sh

‎benchmark/benchmark.sh‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
#!/bin/sh
2+
3+
SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
4+
REPO_DIR=$(CDPATH='' cd -- "$SCRIPT_DIR/.." && pwd)
5+
CC_SH="$REPO_DIR/cc.sh"
6+
7+
if [ ! -f "$CC_SH" ]; then
8+
echo "Cannot find cc.sh at $CC_SH" >&2
9+
exit 1
10+
fi
11+
12+
export SPACK_COMPILER_WRAPPER_PATH="$SCRIPT_DIR"
13+
export SPACK_DEBUG_LOG_DIR="/tmp"
14+
export SPACK_DEBUG_LOG_ID="bench"
15+
export SPACK_SHORT_SPEC="py-torch@2.11.0%gcc@14.2.0 arch=linux-ubuntu24.04-aarch64"
16+
export SPACK_SYSTEM_DIRS="/usr/*|/lib/*"
17+
export SPACK_CXX="true"
18+
export SPACK_CC="true"
19+
export SPACK_FC="true"
20+
export SPACK_F77="true"
21+
export SPACK_CC_LINKER_ARG="-Wl,"
22+
export SPACK_CC_RPATH_ARG="-Wl,-rpath,"
23+
export SPACK_CXX_LINKER_ARG="-Wl,"
24+
export SPACK_CXX_RPATH_ARG="-Wl,-rpath,"
25+
export SPACK_FC_LINKER_ARG="-Wl,"
26+
export SPACK_FC_RPATH_ARG="-Wl,-rpath,"
27+
export SPACK_F77_LINKER_ARG="-Wl,"
28+
export SPACK_F77_RPATH_ARG="-Wl,-rpath,"
29+
export SPACK_CXX_HAS_FRANDOM_SEED=true
30+
export SPACK_DTAGS_TO_ADD="--enable-new-dtags"
31+
export SPACK_DTAGS_TO_STRIP="--disable-new-dtags"
32+
33+
_P="/home/software/spack/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeholder__/__spack_path_placeh/linux-aarch64"
34+
35+
export SPACK_MANAGED_DIRS="$_P/*"
36+
37+
# Hash generator: produces a fake 32-char hex from an index.
38+
_h() {
39+
printf '%032x' "$1"
40+
}
41+
42+
# Build large dependency lists to stress the wrapper.
43+
_store_pkgs="python-3.13.2 py-numpy-2.4.1 py-setuptools-80.7.1 py-pyyaml-6.0.2 openblas-0.3.33 cuda-12.9.1 py-pip-24.2 py-wheel-0.43.0 py-cython-3.0.11 py-pybind11-2.13.6 py-typing-extensions-4.12.2 py-sympy-1.13.3 py-mpmath-1.3.0 py-filelock-3.16.1 py-jinja2-3.1.4 py-markupsafe-2.1.5 py-networkx-3.4.2 py-fsspec-2024.10.0 py-packaging-24.1 py-six-1.16.0 zlib-ng-2.2.2 bzip2-1.0.8 xz-5.4.6 zstd-1.5.6 openssl-3.3.2 sqlite-3.46.1 ncurses-6.5 readline-8.2 libffi-3.4.6 expat-2.6.4 gdbm-1.24 util-linux-uuid-2.40.2 tar-1.35 gettext-0.22.5 libiconv-1.17 pcre2-10.44 ca-certificates-2024 cmake-3.31.0 ninja-1.12.1"
44+
45+
_dep_pkgs="libxfixes-6.0.1 libxdamage-1.1.6 libxshmfence-1.3.2 libxxf86vm-1.1.5 libxcursor-1.2.3 libxcomposite-0.4.6 libxinerama-1.1.5 libxrandr-1.5.4 libxrender-0.9.11 libxext-1.3.6 libxi-1.8.2 libxtst-1.2.5 libxkbcommon-1.7.0 libxkbfile-1.1.3 libxmu-1.2.1 libxt-1.3.0 libxaw-1.0.16 libxpm-3.5.17 libxft-2.3.8 libxss-1.2.4 libx11-1.8.10 libxcb-1.17.0 libxau-1.0.11 libxdmcp-1.1.5 libxres-1.2.2 libxv-1.0.12 libxvmc-1.0.14 libpciaccess-0.18.1 libdrm-2.4.123 mesa-24.2.5 libglvnd-1.7.0 libglx-1.7.0 libegl-1.7.0 libgles-1.7.0 wayland-1.23.1 xorgproto-2024.1 xtrans-1.5.1 fontconfig-2.15.0 freetype-2.13.3 harfbuzz-10.0.1 libpng-1.6.44 libjpeg-turbo-3.0.4 libtiff-4.7.0 libwebp-1.4.0 giflib-5.2.2 librsvg-2.59.0 cairo-1.18.2 pixman-0.43.4 pango-1.54.0 fribidi-1.0.16 graphite2-1.3.14"
46+
47+
_idx=0
48+
_store_include=""
49+
_store_link=""
50+
for _pkg in $_store_pkgs; do
51+
_hash=$(_h $_idx)
52+
_store_include="$_store_include$_P/$_pkg-$_hash/include:"
53+
_store_link="$_store_link$_P/$_pkg-$_hash/lib:"
54+
_idx=$((_idx + 1))
55+
done
56+
57+
_dep_include=""
58+
_dep_link=""
59+
for _pkg in $_dep_pkgs; do
60+
_hash=$(_h $_idx)
61+
_dep_include="$_dep_include$_P/$_pkg-$_hash/include:"
62+
_dep_link="$_dep_link$_P/$_pkg-$_hash/lib:"
63+
_idx=$((_idx + 1))
64+
done
65+
66+
export SPACK_STORE_INCLUDE_DIRS="${_store_include%:}"
67+
export SPACK_STORE_LINK_DIRS="${_store_link%:}"
68+
export SPACK_STORE_RPATH_DIRS="$SPACK_STORE_LINK_DIRS"
69+
70+
export SPACK_INCLUDE_DIRS="${_dep_include%:}"
71+
export SPACK_LINK_DIRS="${_dep_link%:}"
72+
export SPACK_RPATH_DIRS="$SPACK_LINK_DIRS"
73+
74+
export SPACK_COMPILER_IMPLICIT_RPATHS="\
75+
$_P/gcc-16.1.0-a30491defb3c2a8a14f43c8c93ec2ef2/lib"
76+
77+
export SPACK_COMPILER_EXTRA_RPATHS="\
78+
$_P/gcc-runtime-stuff-16.1.0-6bce6552b41bbeed94e4faee72dcbe96/lib"
79+
80+
export SPACK_CFLAGS=""
81+
export SPACK_CXXFLAGS=""
82+
export SPACK_FFLAGS=""
83+
export SPACK_CPPFLAGS=""
84+
export SPACK_LDFLAGS=""
85+
export SPACK_LDLIBS=""
86+
export SPACK_ALWAYS_CFLAGS=""
87+
export SPACK_ALWAYS_CXXFLAGS=""
88+
export SPACK_ALWAYS_CPPFLAGS=""
89+
export SPACK_ALWAYS_FFLAGS=""
90+
export SPACK_TARGET_ARGS_CC=""
91+
export SPACK_TARGET_ARGS_CXX=""
92+
export SPACK_TARGET_ARGS_FORTRAN=""
93+
export SPACK_COMPILER_FLAGS_KEEP=""
94+
export SPACK_COMPILER_FLAGS_REPLACE=""
95+
96+
# Warmup
97+
"$SCRIPT_DIR/g++" -c foo.c -o foo.o
98+
99+
# Do a 1000 runs
100+
N=1000
101+
_start=$(date +%s)
102+
_i=0
103+
while [ $_i -lt $N ]; do
104+
"$SCRIPT_DIR/g++" -c foo.c -o foo.o
105+
_i=$((_i + 1))
106+
done
107+
_end=$(date +%s)
108+
_elapsed=$((_end - _start))
109+
printf '%s runs in %ss (avg %sms)\n' "$N" "$_elapsed" "$((_elapsed * 1000 / N))"

‎benchmark/g++‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
../cc.sh

‎cc.sh‎

Lines changed: 60 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424
# other separators, we set and reset it.
2525
unset IFS
2626

27+
# BEGIN list functions
28+
2729
# Separator for lists whose names end with `_list`.
2830
# We pick the alarm bell character, which is highly unlikely to
2931
# conflict with anything. This is a literal bell character (which
@@ -94,23 +96,6 @@ setsep() {
9496
esac
9597
}
9698

97-
# prepend LISTNAME ELEMENT
98-
#
99-
# Prepend ELEMENT to the list stored in the variable LISTNAME.
100-
# Handles empty lists and single-element lists.
101-
prepend() {
102-
varname="$1"
103-
elt="$2"
104-
105-
if empty "$varname"; then
106-
eval "$varname=\"\${elt}\""
107-
else
108-
# Get the appropriate separator for the list we're appending to.
109-
setsep "$varname"
110-
eval "$varname=\"\${elt}${sep}\${$varname}\""
111-
fi
112-
}
113-
11499
# append LISTNAME ELEMENT [SEP]
115100
#
116101
# Append ELEMENT to the list stored in the variable LISTNAME,
@@ -129,43 +114,76 @@ append() {
129114
fi
130115
}
131116

132-
# extend LISTNAME1 LISTNAME2 [PREFIX]
117+
# extend DST_LISTNAME SRC_LISTNAME [PREFIX]
133118
#
134-
# Append the elements stored in the variable LISTNAME2
135-
# to the list stored in LISTNAME1.
119+
# Append the elements stored in the variable SRC_LISTNAME
120+
# to the list stored in DST_LISTNAME.
136121
# If PREFIX is provided, prepend it to each element.
137122
extend() {
138-
# Figure out the appropriate IFS for the list we're reading.
139-
setsep "$2"
140-
if [ "$sep" != " " ]; then
141-
IFS="$sep"
142-
fi
143-
eval "for elt in \${$2}; do append $1 \"$3\${elt}\"; done"
123+
_dst="$1"
124+
_src="$2"
125+
_prefix="$3"
126+
127+
# Turn source list into positional parameters
128+
setsep "$_src"
129+
[ "$sep" != " " ] && IFS="$sep"
130+
eval "set -- \${$_src}"
144131
unset IFS
132+
133+
[ $# -eq 0 ] && return
134+
135+
setsep "$_dst"; _dst_sep="$sep"
136+
137+
if [ -z "$_prefix" ]; then
138+
# Fast concatenation when no prefix is needed
139+
IFS="$_dst_sep"; _ext_str="$*"; unset IFS
140+
else
141+
_ext_str="${_prefix}$1"
142+
shift
143+
for elt; do
144+
_ext_str="${_ext_str}${_dst_sep}${_prefix}${elt}"
145+
done
146+
fi
147+
148+
eval "$_dst=\"\${$_dst:+\${$_dst}$_dst_sep}\${_ext_str}\""
145149
}
146150

147-
# preextend LISTNAME1 LISTNAME2 [PREFIX]
151+
# preextend DST_LISTNAME SRC_LISTNAME [PREFIX]
148152
#
149-
# Prepend the elements stored in the list at LISTNAME2
150-
# to the list at LISTNAME1, preserving order.
153+
# Prepend the elements stored in the list at SRC_LISTNAME
154+
# to the list at DST_LISTNAME, preserving order.
151155
# If PREFIX is provided, prepend it to each element.
152156
preextend() {
153-
# Figure out the appropriate IFS for the list we're reading.
154-
setsep "$2"
155-
if [ "$sep" != " " ]; then
156-
IFS="$sep"
157-
fi
157+
_dst="$1"
158+
_src="$2"
159+
_prefix="$3"
160+
161+
# Turn source list into positional parameters
162+
setsep "$_src"
163+
[ "$sep" != " " ] && IFS="$sep"
164+
eval "set -- \${$_src}"
165+
unset IFS
158166

159-
# first, reverse the list to prepend
160-
_reversed_list=""
161-
eval "for elt in \${$2}; do prepend _reversed_list \"$3\${elt}\"; done"
167+
[ $# -eq 0 ] && return
162168

163-
# prepend reversed list to preextend in order
164-
IFS="${lsep}"
165-
for elt in $_reversed_list; do prepend "$1" "$3${elt}"; done
166-
unset IFS
169+
setsep "$_dst"; _dst_sep="$sep"
170+
171+
if [ -z "$_prefix" ]; then
172+
# Fast concatenation when no prefix is needed
173+
IFS="$_dst_sep"; _ext_str="$*"; unset IFS
174+
else
175+
_ext_str="${_prefix}$1"
176+
shift
177+
for elt; do
178+
_ext_str="${_ext_str}${_dst_sep}${_prefix}${elt}"
179+
done
180+
fi
181+
182+
eval "$_dst=\"\${_ext_str}\${$_dst:+$_dst_sep\${$_dst}}\""
167183
}
168184

185+
# END list functions
186+
169187
execute() {
170188
# dump the full command if the caller supplies SPACK_TEST_COMMAND=dump-args
171189
if [ -n "${SPACK_TEST_COMMAND=}" ]; then
@@ -979,7 +997,7 @@ esac
979997
if [ -n "$SPACK_CCACHE_BINARY" ]; then
980998
case "$lang_flags" in
981999
C|CXX) # ccache only supports C languages
982-
prepend full_command_list "${SPACK_CCACHE_BINARY}"
1000+
full_command_list="${SPACK_CCACHE_BINARY}${lsep}${full_command_list}"
9831001
# workaround for stage being a temp folder
9841002
# see #3761#issuecomment-294352232
9851003
export CCACHE_NOHASHDIR=yes

0 commit comments

Comments
 (0)