klesh/dotfiles
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

[feature] backup pg / dir all works

Browse Source
This commit is contained in:
Klesh Wong 2021-01-18 00:33:07 +08:00
parent ef7399810c
commit 54726273f1
6 changed files with 95 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
devops/Dockerfile
View File

@ -1,3 +1,10 @@
FROM curlimages/curl
FROM lachlanevenson/k8s-kubectl:v1.20.1
RUN sed -i 's|dl-cdn.alpinelinux.org|mirrors.aliyun.com|g' /etc/apk/repositories
RUN apk update --no-cache \
&& apk add --update --no-cache curl \
&& rm -rf /var/cache/apk/*
WORKDIR /opt/devops
ADD bin .
ENV PATH=/opt/devops:$PATH
ENTRYPOINT ["/usr/bin/env"]
CMD ["ls"]

31
devops/bin/backup-dirs.sh Executable file
View File

@ -0,0 +1,31 @@
#!/bin/sh
set -e
if [ "$#" -lt 2 ] ;then
echo "Keep N days backup of a directory"
echo
echo " Usage $0 <days> <backup-dir> <src-dir ...>"
exit 1
fi
DIR=$(dirname "$(readlink -f "$0")")
DAY=$1
BKD=${2%/}
PTH=$BKD/$(date +%Y%m%d)
shift
shift
echo "start backing up on $(date)"
mkdir -p $PTH
while [ "$#" -gt 0 ] ;do
SRC=$1
echo " backing up $SRC"
tar -zcf "$PTH/$(basename $SRC).tar.gz" "$SRC"
shift
done
echo "start removing older archives"
"$DIR/rm-ndays-ago.sh" "$BKD" "$DAY"
echo "done!"

22
devops/bin/backup-k8s-pgdb.sh Executable file
View File

@ -0,0 +1,22 @@
#!/bin/sh
set -e
if [ "$#" -lt 3 ] ;then
echo "Keep recent N-days backup of postgres on k8s container"
echo
echo " Usage $0 <days> <path/to/backup/directory> <pod-app-label> [container-name] [DBN]"
exit 1
fi
DIR=$(dirname "$(readlink -f "$0")")
DAY=$1
BKD=$2
shift
shift
echo "start backing up on $(date)"
"$DIR/k8s-pgdball.sh" backup "$BKD/$(date +%Y%m%d)" "$@"
echo "start removing older archives"
"$DIR/rm-ndays-ago.sh" "$BKD" "$DAY"
echo "done!"

25
devops/bin/k8s-pgdb.sh
View File

@ -2,32 +2,35 @@
set -e
if [ "$#" -lt 4 ] ;then
echo "Backup postgres database on k8s container with pg_dump and gzip"
if [ "$#" -lt 3 ] ;then
echo "Backup/Restore postgres database on k8s"
echo
echo " Usage $0 <path-to-dbname.gz> <pod-app-label> <container-name> [dbname]"
echo " Usage $0 <backup|restore> <path/to/dbname.sql.gz> <pod-app-label> [container] [dbname]"
exit 1
fi
BACKUPGZ=$2
GZP=$2
APP=$3
CONTAINER=$4
DBNAME=${5-"$(basename "$BACKUPGZ" .sql.gz)"}
CTN=$4
DBN=${5-"$(basename "$GZP" .sql.gz)"}
POD=$(kubectl get pod --selector "app=$APP" | tail +2 | head -n 1 | awk '{print $1}')
POD=$(kubectl get pod --selector "app=$APP" | tail +2 | awk '{print $1}')
if [ -z "$POD" ] ; then
echo not pod found!
exit 1
elif [ "$(echo "$POD" | wc -w)" -gt 1 ] ;then
echo more than one pod!
exit 1
fi
backup () {
kubectl exec "$POD" -c "$CONTAINER" -- pg_dump -U postgres --no-owner "$DBNAME" | gzip > "$BACKUPGZ"
kubectl exec "$POD" -c "$CTN" -- sh -c "pg_dump -U postgres --no-owner $DBN | gzip" > "$GZP"
}
restore () {
kubectl exec "$POD" -c "$CONTAINER" -- psql postgres postgres -c "DROP DATABASE IF EXISTS $DBNAME"
kubectl exec "$POD" -c "$CONTAINER" -- psql postgres postgres -c "CREATE DATABASE $DBNAME"
kubectl exec -i "$POD" -c "$CONTAINER" -- sh -c "gunzip -c | psql $DBNAME postgres" < "$BACKUPGZ"
kubectl exec "$POD" -c "$CTN" -- psql postgres postgres -c "DROP DATABASE IF EXISTS $DBN"
kubectl exec "$POD" -c "$CTN" -- psql postgres postgres -c "CREATE DATABASE $DBN"
kubectl exec "$POD" -c "$CTN" -i -- sh -c "gunzip -c | psql $DBN postgres" < "$GZP"
}

27
devops/bin/k8s-pgdball.sh
View File

@ -2,44 +2,47 @@
set -e
if [ "$#" -lt 4 ] ;then
echo "Backup all postgres databases on k8s container with pg_dump and gzip"
if [ "$#" -lt 3 ] ;then
echo "Backup/Restore all postgres databases on k8s container with pg_dump and gzip"
echo
echo " Usage $0 <path-to-backup-directory> <pod-app-label> <container-name> [dbname]"
echo " Usage $0 <backup|restore> <path/to/backup/directory> <pod-app-label> [container-name] [DBN]"
exit 1
fi
DIR=$(dirname "$(readlink -f "$0")")
BACKUPDIR=$2
BKD=$2
APP=$3
CONTAINER=$4
CTN=$4
POD=$(kubectl get pod --selector "app=$APP" | tail +2 | awk '{print $1}')
if [ -z "$POD" ] ; then
echo not pod found!
exit 1
elif [ "$(echo "$POD" | wc -w)" -gt 1 ] ;then
echo more than one pod!
exit 1
fi
backup () {
mkdir -p "$BACKUPDIR"
for DBNAME in $(
kubectl exec "$POD" -c "$CONTAINER" -- \
mkdir -p "$BKD"
for DBN in $(
kubectl exec "$POD" -c "$CTN" -- \
psql postgres postgres -c "SELECT datname FROM pg_database" | \
tail +3 | head -n -2
); do
case $DBNAME in
case $DBN in
postgres|template0|template1)
;;
*)
echo "backing up $DBNAME to $BACKUPDIR/$DBNAME.sql.gz"
"$DIR/k8s-pgdb.sh" backup "$BACKUPDIR/$DBNAME.sql.gz" "$APP" "$CONTAINER" "$DBNAME"
echo "backing up $DBN to $BKD/$DBN.sql.gz"
"$DIR/k8s-pgdb.sh" backup "$BKD/$DBN.sql.gz" "$APP" "$CTN" "$DBN"
;;
esac
done
}
restore () {
for BACKUPGZ in "$BACKUPDIR"/*.sql.gz ;do
for BACKUPGZ in "$BKD"/*.sql.gz ;do
"$DIR/pgdb.sh" restore "$BACKUPGZ"
done
}

8
devops/bin/rm-ndays-ago.sh
View File

@ -10,11 +10,13 @@ if [ "$#" -lt 2 ] ;then
fi
TARGET_DIR=${1%/}
THRES_DATE=$(date -d "$2 days ago" +%Y%m%d)
THRES_DATE=$(date -D "%S" -d $(( $(date +%s) - $2 * 24 * 3600 ))"" +%Y%m%d)
echo date prior to $THRES_DATE will be deleted
for entry in $(ls "$TARGET_DIR") ;do
DATE=$(echo "$entry" | grep -oP '\b(\d{8})\b')
DATE=$(echo "$entry" | grep -oE '[0-9]{8}')
if [ -n "DATE" ] && [ "$DATE" -lt "$THRES_DATE" ] ;then
rm -f $TARGET_DIR/$entry
echo removeing $TARGET_DIR/$entry
rm -rf $TARGET_DIR/$entry
fi
done