path: root/create-squashfs-snapshot

#!/usr/bin/env bash

# Copyright (c) 2014-2015, Michał Górny
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.

shopt -s nullglob
set -e -x
date +%s.%N

# == config ==
# filled with gentoo-specific details, change at will

source /usr/local/bin/mastermirror/rsync-gen.vars

# Final snapshots and deltas get mirrored out here.
mirrordir=${UPLOAD}/squashfs
# We keep reverse deltas around here to allow handling past snapshots
# without having to keep them in full size.
revdeltadir=${BASE}/squashfs-tmp
# This is where the master (unpacked) copy of the repository is located.
repodir=${FINALDIR_repo_gentoo}

# GPG key ID to sign with
signkeyid="DCD05B71EAB94199527F44ACDB6B8C1F96D8BF6D"

# Deltas to keep before cleanup
cleanupno=180

# == config ends here ==

if [[ ! -d ${revdeltadir} ]]; then
	mkdir -p "${revdeltadir}"
fi

# Squashdelta only supports lzo and lz4
algo_lzo=( -comp lzo -Xcompression-level 4 )
algo_xz=( -comp xz )
algo_LIST=( lzo xz )
mksquashfs_options=(
	-no-xattrs -force-uid portage -force-gid portage
)

[[ -d ${mirrordir} ]]
[[ -d ${revdeltadir} ]]
[[ -d ${repodir} ]]

reponame=$(<"${repodir}"/profiles/repo_name)

[[ ${reponame} ]]

tempdir=$(mktemp -d)
[[ -z $tempdir ]] && echo "Failed to create tempdir" 1>&2 && exit 97
trap 'rm -rf "${tempdir}"' SIGINT SIGTERM EXIT

# Build exclusion list
EXCLUSION_LIST="$(mktemp -p "${tempdir}" squashfs-exclude.XXXXXXXXXX)"
"$(dirname "$0")"/print-exclusion-list.sh "${repodir}" >"${EXCLUSION_LIST}"
mksquashfs_options+=( -ef "${EXCLUSION_LIST}" )


for algo in "${algo_LIST[@]}" ; do
	ext=".${algo}.sqfs"

	snapshots=( "${mirrordir}"/${reponame}-*${ext} )

	if [[ ${snapshots[@]} ]]; then
		yesterdaysnap=${snapshots[-1]}
	  if [[ ${yesterdaysnap} == *-current${ext} ]]; then
	    yesterdaysnap=$(readlink -m "${yesterdaysnap}")
	  fi
		yesterday=${yesterdaysnap#*/${reponame}-}
		yesterday=${yesterday%${ext}}
	fi

	# get date from the repo, move it few hours back to ensure
	# it is 'previous day', alike .tar snapshots
	today=$(date --date="$(<"${repodir}"/metadata/timestamp.chk ) - 6 hours" +%Y%m%d)
	todaysnap_file=${reponame}-${today}${ext}
	todaysnap=${mirrordir}/${todaysnap_file}

	if [[ ! -f ${todaysnap} ]]; then
		# take today's snapshot
		tmp="algo_${algo}[@]"
		file="${tempdir}/${reponame}-${today}${ext}"
		mksquashfs "${repodir}" "${file}" "${mksquashfs_options[@]}" "${!tmp}"
		mv "${file}" "${mirrordir}/"
	fi

	# Deltas are not supported for XZ
	if false && [[ "${algo}" != "xz" ]]; then
		if [[ ${yesterday} ]]; then
			# create rev-delta from today to yesterday
			squashdelta "${todaysnap}" "${yesterdaysnap}" \
				"${revdeltadir}/${reponame}-${today}-${yesterday}.${algo}.sqdelta"

			# create deltas from previous days to today
			revdeltas=( "${revdeltadir}"/*.${algo}.sqdelta )
			lastdelta=$(( ${#revdeltas[@]} - cleanupno ))
			for (( i = ${#revdeltas[@]} - 1; i >= 0; i-- )); do
				[[ ${i} != "${lastdelta}" ]] || break

				r=${revdeltas[${i}]}
				ldate=${r#*/${reponame}-}
				rdate=${ldate%.${algo}.sqdelta}
				ldate=${ldate%-*}
				rdate=${rdate#*-}

				# ldate = newer, rdate = older

				rsnap=${tempdir}/${reponame}-${rdate}${ext}
				if [[ ${rdate} == "${yesterday}" ]]; then
					# we have yesterday's snapshot already, so use it
					cp "${yesterdaysnap}" "${rsnap}"
				else
					# otherwise, we need to reconstruct the snap
					lsnap=${tempdir}/${reponame}-${ldate}${ext}
					if [[ ${ldate} == "${yesterday}" ]]; then
						cp "${yesterdaysnap}" "${lsnap}"
					fi

					squashmerge "${lsnap}" "${r}" "${rsnap}"
					rm "${lsnap}"
				fi

				squashdelta "${rsnap}" "${todaysnap}" "${tempdir}/${reponame}-${rdate}-${today}.${algo}.sqdelta"
				mv "${tempdir}/${reponame}-${rdate}-${today}.${algo}.sqdelta" "${mirrordir}/"
				# remove the last snapshot used
				rm "${rsnap}"
			done


			# finally, clean up the old deltas
			rm -f "${mirrordir}/${reponame}"-*-"${yesterday}.${algo}.sqdelta"
		fi
	fi

	# == here ${PWD} becomes ${mirrordir} ==
	cd "${mirrordir}"

	# create convenience -current symlink, for direct fetching
	ln -s -f "${reponame}-${today}${ext}" "${reponame}-current${ext}"
done

# create checksums for snapshot and deltas
# OLD LOGIC, that scans entire 18GB; with 18GB of data this added 2 minutes of
# runtime onto a script that is otherwise under 30 seconds.
#date +ts-old-checksum-start=%s.%N
#ls -d -- *.sqfs *.sqdelta \
#| xargs sha512sum -- \
#| sort -k +2 \
#| gpg \
#	--batch \
#	--yes \
#	-u "${signkeyid}" \
#	--clearsign \
#	--comment "Current: gentoo-${today}" \
#	--output sha512sum.txt.tmp \
#	/dev/stdin
#mv sha512sum.txt.tmp sha512sum.txt
#date +ts-old-checksum-end=%s.%N

# NEW LOGIC, that tries to re-use signed checksums
# Helper func for signing.
sign_prefix() {
	prefix=$1
	d=${prefix}.sha512sum.txt

	# Might be symlink OR real file.
	find . \
			-name "${prefix}*" \
			-a \( -name "*.sqfs" -o -name '*.sqdelta' \) \
			-printf '%f\n' \
	| xargs sha512sum \
	| gpg --yes -u "${signkeyid}" --clearsign \
	--comment "Daily: ${prefix}" \
	--output "${d}.tmp" \
	/dev/stdin

	mv "${d}".tmp "${d}"
}

date +ts-new-checksum-start=%s.%N
# 1. Create per-day checksums, with dates in the filenames; only if they do NOT
# exist.
find . -maxdepth 1 -mindepth 1 -name 'gentoo-*sqfs' -type f -printf '%f\n' \
| cut -d. -f1 \
| sort \
| uniq \
| perl -lne 'print $_ unless -e $_.".sha512sum.txt"' \
| while read -r _prefix ; do
  sign_prefix "$_prefix"
done
# 2. Always re-sign the -current symlink.
sign_prefix "gentoo-current"

# 3. Re-verify each existing file
find . -maxdepth 1 -mindepth 1 -name 'gentoo-*sha512sum.txt' -printf '%f\n' \
| while read -r sigfile ; do
	rm -f "${tempdir}"/"${sigfile}".combine*
	# FUTURE: it would be wonderful to have a better interface to GPG here that
	# only sent the output if the signature was valid and matched the specified
	# key.
	# gpgv doesn't have assert-signer
	if gpg --verify \
			--batch \
			--assert-signer "${signkeyid}" \
			--trusted-key "${signkeyid}" \
			--output "${tempdir}/${sigfile}".combine-tmp \
			--status-fd 3 \
			2>"${tempdir}/${sigfile}".stderr \
			3>"${tempdir}/${sigfile}".status-fd \
			"$sigfile" \
		; then
		mv "${tempdir}/${sigfile}".combine-tmp "${tempdir}/${sigfile}".combine-verified
	else
		echo "FATAL: $sigfile verification failed" 1>&2
		cat "${tempdir}/${sigfile}".stderr
		cat "${tempdir}/${sigfile}".status-fd
	fi
done

# 4. Combine the verified files and sign the combined output.
find "$tempdir" -name 'gentoo-*combine-verified' \
	| xargs --no-run-if-empty cat  \
	| sort -k +2 \
	| gpg \
	--batch \
	--yes \
	-u "${signkeyid}" \
	--clearsign \
	--comment "Current: gentoo-${today}" \
	--output "${tempdir}"/sha512sum.txt.tmp \
	/dev/stdin
# Move the final file into place
mv "${tempdir}"/sha512sum.txt.tmp sha512sum.txt
date +ts-new-checksum-done=%s.%N
date +ts-last=%s.%N