From 54726273f18f5b0b7923204f906338ebfa8d1d0c Mon Sep 17 00:00:00 2001 From: Klesh Wong Date: Mon, 18 Jan 2021 00:33:07 +0800 Subject: [PATCH] [feature] backup pg / dir all works --- devops/Dockerfile | 9 ++++++++- devops/bin/backup-dirs.sh | 31 +++++++++++++++++++++++++++++++ devops/bin/backup-k8s-pgdb.sh | 22 ++++++++++++++++++++++ devops/bin/k8s-pgdb.sh | 25 ++++++++++++++----------- devops/bin/k8s-pgdball.sh | 27 +++++++++++++++------------ devops/bin/rm-ndays-ago.sh | 8 +++++--- 6 files changed, 95 insertions(+), 27 deletions(-) create mode 100755 devops/bin/backup-dirs.sh create mode 100755 devops/bin/backup-k8s-pgdb.sh diff --git a/devops/Dockerfile b/devops/Dockerfile index 63d80cb..a1fe62a 100644 --- a/devops/Dockerfile +++ b/devops/Dockerfile @@ -1,3 +1,10 @@ -FROM curlimages/curl +FROM lachlanevenson/k8s-kubectl:v1.20.1 +RUN sed -i 's|dl-cdn.alpinelinux.org|mirrors.aliyun.com|g' /etc/apk/repositories +RUN apk update --no-cache \ + && apk add --update --no-cache curl \ + && rm -rf /var/cache/apk/* WORKDIR /opt/devops ADD bin . +ENV PATH=/opt/devops:$PATH +ENTRYPOINT ["/usr/bin/env"] +CMD ["ls"] diff --git a/devops/bin/backup-dirs.sh b/devops/bin/backup-dirs.sh new file mode 100755 index 0000000..e548e66 --- /dev/null +++ b/devops/bin/backup-dirs.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +set -e + +if [ "$#" -lt 2 ] ;then + echo "Keep N days backup of a directory" + echo + echo " Usage $0 " + exit 1 +fi + +DIR=$(dirname "$(readlink -f "$0")") +DAY=$1 +BKD=${2%/} +PTH=$BKD/$(date +%Y%m%d) +shift +shift + +echo "start backing up on $(date)" +mkdir -p $PTH +while [ "$#" -gt 0 ] ;do + SRC=$1 + echo " backing up $SRC" + tar -zcf "$PTH/$(basename $SRC).tar.gz" "$SRC" + shift +done + +echo "start removing older archives" +"$DIR/rm-ndays-ago.sh" "$BKD" "$DAY" + +echo "done!" diff --git a/devops/bin/backup-k8s-pgdb.sh b/devops/bin/backup-k8s-pgdb.sh new file mode 100755 index 0000000..5285e3f --- /dev/null +++ b/devops/bin/backup-k8s-pgdb.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +set -e + +if [ "$#" -lt 3 ] ;then + echo "Keep recent N-days backup of postgres on k8s container" + echo + echo " Usage $0 [container-name] [DBN]" + exit 1 +fi + +DIR=$(dirname "$(readlink -f "$0")") +DAY=$1 +BKD=$2 +shift +shift + +echo "start backing up on $(date)" +"$DIR/k8s-pgdball.sh" backup "$BKD/$(date +%Y%m%d)" "$@" +echo "start removing older archives" +"$DIR/rm-ndays-ago.sh" "$BKD" "$DAY" +echo "done!" diff --git a/devops/bin/k8s-pgdb.sh b/devops/bin/k8s-pgdb.sh index a045072..bae0c73 100755 --- a/devops/bin/k8s-pgdb.sh +++ b/devops/bin/k8s-pgdb.sh @@ -2,32 +2,35 @@ set -e -if [ "$#" -lt 4 ] ;then - echo "Backup postgres database on k8s container with pg_dump and gzip" +if [ "$#" -lt 3 ] ;then + echo "Backup/Restore postgres database on k8s" echo - echo " Usage $0 [dbname]" + echo " Usage $0 [container] [dbname]" exit 1 fi -BACKUPGZ=$2 +GZP=$2 APP=$3 -CONTAINER=$4 -DBNAME=${5-"$(basename "$BACKUPGZ" .sql.gz)"} +CTN=$4 +DBN=${5-"$(basename "$GZP" .sql.gz)"} +POD=$(kubectl get pod --selector "app=$APP" | tail +2 | head -n 1 | awk '{print $1}') -POD=$(kubectl get pod --selector "app=$APP" | tail +2 | awk '{print $1}') if [ -z "$POD" ] ; then echo not pod found! exit 1 +elif [ "$(echo "$POD" | wc -w)" -gt 1 ] ;then + echo more than one pod! + exit 1 fi backup () { - kubectl exec "$POD" -c "$CONTAINER" -- pg_dump -U postgres --no-owner "$DBNAME" | gzip > "$BACKUPGZ" + kubectl exec "$POD" -c "$CTN" -- sh -c "pg_dump -U postgres --no-owner $DBN | gzip" > "$GZP" } restore () { - kubectl exec "$POD" -c "$CONTAINER" -- psql postgres postgres -c "DROP DATABASE IF EXISTS $DBNAME" - kubectl exec "$POD" -c "$CONTAINER" -- psql postgres postgres -c "CREATE DATABASE $DBNAME" - kubectl exec -i "$POD" -c "$CONTAINER" -- sh -c "gunzip -c | psql $DBNAME postgres" < "$BACKUPGZ" + kubectl exec "$POD" -c "$CTN" -- psql postgres postgres -c "DROP DATABASE IF EXISTS $DBN" + kubectl exec "$POD" -c "$CTN" -- psql postgres postgres -c "CREATE DATABASE $DBN" + kubectl exec "$POD" -c "$CTN" -i -- sh -c "gunzip -c | psql $DBN postgres" < "$GZP" } diff --git a/devops/bin/k8s-pgdball.sh b/devops/bin/k8s-pgdball.sh index adfca05..f3f442a 100755 --- a/devops/bin/k8s-pgdball.sh +++ b/devops/bin/k8s-pgdball.sh @@ -2,44 +2,47 @@ set -e -if [ "$#" -lt 4 ] ;then - echo "Backup all postgres databases on k8s container with pg_dump and gzip" +if [ "$#" -lt 3 ] ;then + echo "Backup/Restore all postgres databases on k8s container with pg_dump and gzip" echo - echo " Usage $0 [dbname]" + echo " Usage $0 [container-name] [DBN]" exit 1 fi DIR=$(dirname "$(readlink -f "$0")") -BACKUPDIR=$2 +BKD=$2 APP=$3 -CONTAINER=$4 +CTN=$4 POD=$(kubectl get pod --selector "app=$APP" | tail +2 | awk '{print $1}') if [ -z "$POD" ] ; then echo not pod found! exit 1 +elif [ "$(echo "$POD" | wc -w)" -gt 1 ] ;then + echo more than one pod! + exit 1 fi backup () { - mkdir -p "$BACKUPDIR" - for DBNAME in $( - kubectl exec "$POD" -c "$CONTAINER" -- \ + mkdir -p "$BKD" + for DBN in $( + kubectl exec "$POD" -c "$CTN" -- \ psql postgres postgres -c "SELECT datname FROM pg_database" | \ tail +3 | head -n -2 ); do - case $DBNAME in + case $DBN in postgres|template0|template1) ;; *) - echo "backing up $DBNAME to $BACKUPDIR/$DBNAME.sql.gz" - "$DIR/k8s-pgdb.sh" backup "$BACKUPDIR/$DBNAME.sql.gz" "$APP" "$CONTAINER" "$DBNAME" + echo "backing up $DBN to $BKD/$DBN.sql.gz" + "$DIR/k8s-pgdb.sh" backup "$BKD/$DBN.sql.gz" "$APP" "$CTN" "$DBN" ;; esac done } restore () { - for BACKUPGZ in "$BACKUPDIR"/*.sql.gz ;do + for BACKUPGZ in "$BKD"/*.sql.gz ;do "$DIR/pgdb.sh" restore "$BACKUPGZ" done } diff --git a/devops/bin/rm-ndays-ago.sh b/devops/bin/rm-ndays-ago.sh index cc96450..bb448dc 100755 --- a/devops/bin/rm-ndays-ago.sh +++ b/devops/bin/rm-ndays-ago.sh @@ -10,11 +10,13 @@ if [ "$#" -lt 2 ] ;then fi TARGET_DIR=${1%/} -THRES_DATE=$(date -d "$2 days ago" +%Y%m%d) +THRES_DATE=$(date -D "%S" -d $(( $(date +%s) - $2 * 24 * 3600 ))"" +%Y%m%d) +echo date prior to $THRES_DATE will be deleted for entry in $(ls "$TARGET_DIR") ;do - DATE=$(echo "$entry" | grep -oP '\b(\d{8})\b') + DATE=$(echo "$entry" | grep -oE '[0-9]{8}') if [ -n "DATE" ] && [ "$DATE" -lt "$THRES_DATE" ] ;then - rm -f $TARGET_DIR/$entry + echo removeing $TARGET_DIR/$entry + rm -rf $TARGET_DIR/$entry fi done