forked from ClickHouse/ClickHouse
-
Notifications
You must be signed in to change notification settings - Fork 3
Expand file tree
/
Copy pathbuild
More file actions
executable file
·286 lines (258 loc) · 10.8 KB
/
Copy pathbuild
File metadata and controls
executable file
·286 lines (258 loc) · 10.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
#!/usr/bin/env bash
set -e
set -o pipefail
# Avoid dependency on locale
LC_ALL=C
PKG_ROOT='root'
DEB_ARCH=${DEB_ARCH:-amd64}
SANITIZER=${SANITIZER:-""}
SOURCE=${SOURCE:-$PKG_ROOT}
cd "$(dirname "${BASH_SOURCE[0]}")"
HELP="${0} [--test] [--rpm] [-h|--help]
--test - adds '+test' prefix to version
--apk - build APK packages
--archlinux - build archlinux packages
--rpm - build RPM packages
--tgz - build tarball package
--deb - build deb package
--help - show this help and exit
Used envs:
DEB_ARCH='${DEB_ARCH}'
OUTPUT_DIR='${OUTPUT_DIR}' - where the artifact will be placed
SANITIZER='${SANITIZER}' - if any sanitizer is used, affects version string
SOURCE='${SOURCE}' - directory with sources tree
VERSION_STRING='${VERSION_STRING}' - the package version to overwrite
"
CLICKHOUSE_VERSION_STRING=${VERSION_STRING}
export CLICKHOUSE_VERSION_STRING
while [[ $1 == --* ]]
do
case "$1" in
--test )
VERSION_POSTFIX+='+test'
shift ;;
--deb )
MAKE_DEB=1
shift ;;
--apk )
MAKE_APK=1
shift ;;
--archlinux )
MAKE_ARCHLINUX=1
shift ;;
--rpm )
MAKE_RPM=1
shift ;;
--tgz )
MAKE_TGZ=1
shift ;;
--help )
echo "$HELP"
exit ;;
* )
echo "Unknown option $1"
exit 2 ;;
esac
done
function deb2tgz {
local FILE PKG_NAME PKG_DIR PKG_PATH TARBALL
FILE=$1
PKG_NAME=${FILE##*/}; PKG_NAME=${PKG_NAME%%_*}
PKG_DIR="$PKG_NAME-$CLICKHOUSE_VERSION_STRING"
PKG_PATH="$OUTPUT_DIR/$PKG_DIR"
TARBALL="$OUTPUT_DIR/$PKG_DIR-$DEB_ARCH.tgz"
rm -rf "$PKG_PATH"
dpkg-deb -R "$FILE" "$PKG_PATH"
mkdir -p "$PKG_PATH/install"
cat > "$PKG_PATH/install/doinst.sh" << 'EOF'
#!/bin/sh
set -e
SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
# Moving a file replaces the destination inode with the one from the unpack tree, so the
# installed file inherits the SELinux label of the temporary directory instead of the one
# the final path should have. Restore the labels from the policy where SELinux is active.
RESTORECON=""
if command -v selinuxenabled > /dev/null 2>&1 && selinuxenabled > /dev/null 2>&1 && command -v restorecon > /dev/null 2>&1; then
RESTORECON="restorecon"
fi
# Extended attributes live on the inode as well, and unlike the SELinux label they cannot be
# recreated from a policy, so a destination carrying them is written through instead of being
# renamed over. `getfattr` reports them where it is available.
GETFATTR=""
if command -v getfattr > /dev/null 2>&1; then
GETFATTR="getfattr"
fi
# Whether the file already at the destination has to survive the installation as the same
# inode. Everything else - and that is the common case, including every fresh installation -
# is replaced by a rename, which needs no additional disk space.
keep_destination_inode () {
# An existing symlink at the destination is written through, the way copying did, so that
# an installation keeping its configs or binaries elsewhere and linking to them keeps the
# indirection after an upgrade. A symlink with no target yet is handled separately in the
# loop below, because there is no inode to write through in that case.
if [ -L "$1" ] && [ -e "$1" ]; then
return 0
fi
if [ ! -e "$1" ]; then
return 1
fi
# Only a regular file can be written through.
if [ ! -f "$1" ]; then
return 1
fi
listing=$(ls -ld -- "$1")
# Another hard link to the destination would keep pointing at the pre-upgrade contents
# after a rename, while writing through updates every link.
if [ "$(printf '%s\n' "$listing" | awk '{print $2}')" -gt 1 ]; then
return 0
fi
# An ACL is reported by `ls` as a `+` after the permissions.
case "$(printf '%s\n' "$listing" | awk '{print $1}')" in
*+) return 0 ;;
esac
if [ -z "$GETFATTR" ]; then
# Without `getfattr` there is no portable way to tell whether the destination carries
# extended attributes, so an existing regular file conservatively keeps its inode.
# The disk-space saving is unaffected: it comes from the fresh installation, where no
# destination exists and the rename path is taken above.
return 0
fi
# The SELinux label is skipped: it is not carried over by a rename either, but it is
# recreated from the policy by `restorecon` below, and on an SELinux system every
# single file has one, which would make the rename path unreachable.
attributes=$($GETFATTR --absolute-names --match=- --dump -- "$1" 2> /dev/null | grep -v -e '^#' -e '^$' -e '^security\.selinux=' || true)
if [ -n "$attributes" ]; then
return 0
fi
return 1
}
# The path a chain of symlinks finally points at. Every hop is followed, and a relative hop is
# resolved against the directory of the link it was read from, so that a destination pointing
# at another symlink keeps every level of the indirection instead of only the first one.
resolve_symlink_chain () {
path="$1"
hops=0
while [ -L "$path" ]; do
hops=$((hops + 1))
if [ "$hops" -gt 40 ]; then
printf '%s\n' "Too many levels of symbolic links at $1" >&2
return 1
fi
target=$(readlink -- "$path")
case "$target" in
/*) path="$target" ;;
*) path="$(dirname -- "$path")/$target" ;;
esac
done
printf '%s\n' "$path"
}
for filepath in `find $SCRIPTPATH/.. -type f -or -type l | grep -v "\.\./install/"`; do
destpath=${filepath##$SCRIPTPATH/..}
mkdir -p $(dirname "$destpath")
if [ ! -L "$filepath" ] && [ -L "$destpath" ] && [ ! -e "$destpath" ]; then
# A symlink whose target does not exist yet has to survive the installation as well,
# but there is no destination inode to write through: a shell redirection would create
# the target with the `umask` of the installation and with no relation to the mode of
# the packaged file, which would install a binary non-executable. Rename the packaged
# file to where the link points instead, so that the link keeps its indirection and the
# installed file gets the mode, the ownership and the label every installed file has.
# The whole chain is followed: a link pointing at another link has to keep both, the
# way writing through the destination did, so only the final referent is created.
linktarget=$(resolve_symlink_chain "$destpath")
linkparent=$(dirname -- "$linktarget")
if [ ! -d "$linkparent" ]; then
# The directory the link points into is not created here. It would be created
# with the `umask` of the installation and with no defined ownership, so the
# installed file could end up unreachable while the installation reports
# success. Writing through the link failed loudly when that directory was
# missing, and pointing a destination at a directory that does not exist stays
# an error the administrator has to see and fix.
printf '%s\n' "Cannot install $destpath: it points at $linktarget, but the directory $linkparent does not exist" >&2
exit 1
fi
mv -f "$filepath" "$linktarget"
chown -h root:root "$linktarget"
if [ -n "$RESTORECON" ]; then
$RESTORECON "$linktarget"
fi
elif [ ! -L "$filepath" ] && keep_destination_inode "$destpath"; then
# The destination keeps its inode, so its ownership, permissions, attributes and every
# other hard link to it stay valid after the upgrade, as they did when copying.
cat "$filepath" > "$destpath"
rm -f "$filepath"
else
# Move instead of copy: copying requires twice the disk space, which is several
# gigabytes for the packages with debug symbols. Moving also succeeds when the
# destination is a running executable, while copying over it fails with ETXTBSY.
mv -f "$filepath" "$destpath"
# The tarball carries the ownership of the user who built it, while the installed
# files have to belong to root, as they do in the deb and rpm packages.
chown -h root:root "$destpath"
if [ -n "$RESTORECON" ]; then
$RESTORECON "$destpath"
fi
fi
done
EOF
chmod +x "$PKG_PATH/install/doinst.sh"
if [ -f "$PKG_PATH/DEBIAN/postinst" ]; then
# we don't need debconf source in doinst in any case
tail +2 "$PKG_PATH/DEBIAN/postinst" | grep -v debconf/confmodule >> "$PKG_PATH/install/doinst.sh"
fi
rm -rf "$PKG_PATH/DEBIAN"
if [ -f "/usr/bin/pigz" ]; then
tar --use-compress-program=pigz -cf "$TARBALL" -C "$OUTPUT_DIR" "$PKG_DIR"
else
tar -czf "$TARBALL" -C "$OUTPUT_DIR" "$PKG_DIR"
fi
# Cut the $OUTPUT_DIR/ from the sha512sum output to make it universal
sha512sum "$TARBALL" | sed "s|$OUTPUT_DIR/||" > "$TARBALL".sha512
rm -r "$PKG_PATH"
}
# Build options
if [ -n "$SANITIZER" ]; then
if [[ "$SANITIZER" == "address" ]]; then VERSION_POSTFIX+="+asan"
elif [[ "$SANITIZER" == "thread" ]]; then VERSION_POSTFIX+="+tsan"
elif [[ "$SANITIZER" == "memory" ]]; then VERSION_POSTFIX+="+msan"
elif [[ "$SANITIZER" == "undefined" ]]; then VERSION_POSTFIX+="+ubsan"
else
echo "Unknown value of SANITIZER variable: $SANITIZER"
exit 3
fi
elif [[ $BUILD_TYPE == 'debug' ]]; then
VERSION_POSTFIX+="+debug"
elif [[ $BUILD_TYPE =~ 'coverage' ]]; then
VERSION_POSTFIX+="+coverage"
elif [[ $BUILD_TYPE == 'cfi' ]]; then
VERSION_POSTFIX+="+cfi"
fi
if [[ "$PKG_ROOT" != "$SOURCE" ]]; then
# packages are built only from PKG_SOURCE
rm -rf "./$PKG_ROOT"
ln -sf "$SOURCE" "$PKG_SOURCE"
fi
CLICKHOUSE_VERSION_STRING+=$VERSION_POSTFIX
echo -e "\nCurrent version is $CLICKHOUSE_VERSION_STRING"
for config in clickhouse*.yaml; do
if [ -n "$MAKE_DEB" ] || [ -n "$MAKE_TGZ" ]; then
echo "Building deb package for $config"
PKG_PATH=$(nfpm package --target "$OUTPUT_DIR" --config "$config" --packager deb | tee /dev/stderr | grep "created package:" | sed 's/.*created package: //')
fi
if [ -n "$MAKE_APK" ]; then
echo "Building apk package for $config"
nfpm package --target "$OUTPUT_DIR" --config "$config" --packager apk
fi
if [ -n "$MAKE_ARCHLINUX" ]; then
echo "Building archlinux package for $config"
nfpm package --target "$OUTPUT_DIR" --config "$config" --packager archlinux
fi
if [ -n "$MAKE_RPM" ]; then
echo "Building rpm package for $config"
nfpm package --target "$OUTPUT_DIR" --config "$config" --packager rpm
fi
if [ -n "$MAKE_TGZ" ]; then
echo "Building tarball for $config"
deb2tgz "$PKG_PATH"
fi
done
# vim: ts=4: sw=4: sts=4: expandtab