User Tools

Site Tools


linux:purging_script

This is an old revision of the document!


Description

Have you ever fight with logrotate configuration files? Did you survived? Cool, this document is not for you xD This is script is a very simple method to drop unnecessary files from your filesystems (logs?) and rotate them so they don't growth forever :-P

This script write 2 logs each time :

  • $(dirname purgefiles.sh)/logs/purgefiles$(date +%Y%m%d%H%M).log * $(dirname purgefiles.sh)/logs/purgefiles$(date +%Y%m%d%H%M).err

So you will have detailed the script actions+errors there

Instructions

Usage

/home/scripts/purge_files/purge_files.sh

Configuration

It's a file-based configuration, the variable inside the script which sets the file is:

CONFIGFILE=/home/scripts/purge_files/purge_files.config

Config file sytax

Each line in the config file is set of file to be analysed and purged/rotated, the syntax is:

#path,mask,action,param1,param2,param3

Where:

  • Path is the folder from which the script begin searching (serching with find)
  • Mask: file mask for matching files, the script uses find -name (Ex: *gz, *log, *txt)
  • Action: Any of the avalaible actions detailed here down.

Y las opciones disponibles son (explicadas como parámetros de find)

Actions

path mask action param1 param2 param3
/path ? bydate mtime maxdepth N/A
/path ? bymin mmin maxdepth N/A
/path ? rotate size maxdepth N/A
/path ? rotatezip size maxdepth gzip parameters
/path ? zip mtime maxdepth gzip parameters
/path ? dirbydate mtime maxdepth N/A
/path ? dirbymin mmin maxdepth N/A

Description:

  • bydate : Delete files based on modification time (find -mtime +X -type f)
  • bymin : Drop files based on modification minute (find -mmin +X -type f)
  • dirbydate : Drop folders based on modification time. (find -mtime +X -type d)
  • dirbymin : Drop folders based on modification minute (find -mmin +X -type d)
  • rotate : Rotate the file (cat file > file.date && > file)
  • rotatezip : Rorate and compress the file (gzip -c file > file.date.gz && > file)
  • zip : Compress the file (gzip file)

Sample Config

#path,mask,action,param1,param2,param3
# Allways purge the script logs
/home/scripts/purge_files/logs,*.log,bydate,+4,
/home/scripts/purge_files/logs,*.err,bydate,+4,

# Target: nginx logs
/var/log/nginx,*.gz,bydate,+20,,

# Objetivo : Oracle listerner trace files
/u01/app/oracle/diag/tnslsnr,*trc,bydate,+15,4,
/u01/app/oracle/diag/tnslsnr,*trm,bydate,+15,4,

Script Code

purge_files.sh
#!/bin/bash
# 
# (C) dodger@ciberterminal.net
# 
 
 
# Exit codes:
#  1 : No config file
#  2 : Wrong option in the config file
#  3 : 
#  4 : 
#  5 : 
#  6 : 
 
########################################################################
#
# CONSTANTS
#
########################################################################
 
# colors
LIGHTGREEN="\033[1;32m"
LIGHTRED="\033[1;31m"
WHITE="\033[0;37m"
RESET="\033[0;00m"
 
 
########################################################################
#
# / CONSTANTS
#
########################################################################
 
 
 
########################################################################
#
# VARIABLES
#
########################################################################
 
MYDATE=$(date +%Y%m%d%H%M)
CONFIGFILE=/home/scripts/purge_files/purge_files.config
LOGDIR=$(dirname $0)/logs/
 
########################################################################
#
# / VARIABLES
#
########################################################################
 
 
########################################################################
#
# FUNCTIONS
#
########################################################################
 
 
usage()
{
    printf "%s${LIGHTRED}USAGE:${RESET}
    $0
 
    PLEASE READ
    https://sites.google.com/a/voxelgroup.net/sistemas/oracle/documentacion-tecnica/rotado-de-logs-y-trazas\n"
    # VERY INITIAL CHECKS
}
 
remove_files()
{
    local AUX=""
    local OPT="-${1}"
    local DIR="${2}"
    if [[ "${DIR,,}" = "dir" ]] ; then
        local TYPEOF="-type d"
        local FINDCMD="-print | xargs rm -fr"
        local MESSAGE="$(date +%Y%m%d%H%M%S) : Deleting folders inside ${FOLDER}, with filemask ${MASK}, ${OPT} ${PARAM1}"
    else
        local TYPEOF="-type f"
        local FINDCMD="-delete"
        local MESSAGE="$(date +%Y%m%d%H%M%S) : Deleting files on ${FOLDER}, with filemask ${MASK}, ${OPT} ${PARAM1}"
    fi
 
    [[ "${PARAM2}" ]] && AUX="-maxdepth ${PARAM2}" && MESSAGE="%s${MESSAGE} and maxdepth ${PARAM2}"
    printf "%s${MESSAGE}\n"
    eval find ${FOLDER} ${AUX} -name "${MASK}" ${TYPEOF} ${OPT} ${PARAM1} ${FINDCMD}
}
 
rotate_files()
{
    local USEZIP="$1"
    local MESSAGE="$(date +%Y%m%d%H%M%S) : Rotating files on ${FOLDER}, with filemask ${MASK}, -mtime ${PARAM1}"
    [[ "${PARAM2}" ]] && AUX="-maxdepth ${PARAM2}" && MESSAGE="${MESSAGE}, maxdepth ${PARAM2}"
    printf "%s${MESSAGE}\n"
#     printf "%s$(date +%Y%m%d%H%M%S) : Rotating files on ${FOLDER}, with filemask ${MASK}, -mtime ${PARAM1}"
#     [[ "${PARAM2}" ]] && AUX="-maxdepth ${PARAM2}" && printf "%s, maxdepth ${PARAM2}"
 
    case $USEZIP in
        "zip" )
            [[ ! "${PARAM3}" ]] && PARAM3="-9fv"
            local MOVECMD="gzip -c ${PARAM3}"
            local DSTFILE="${MYDATE}.gz"
            printf "%s and gzipping with ${PARAM3}\n"
        ;;
        "nozip" )
            local MOVECMD="cat"
            local DSTFILE="${MYDATE}"
            printf "%s\n"
        ;;
        * )
            printf "%s${LIGHTRED}ERROR ON rotate_files function${RESET}"
            return 1
        ;;
    esac
    while read FILE ; do
        echo ${FILE}
        ${MOVECMD} "${FILE}" > "${FILE}.${DSTFILE}"
         > ${FILE}
    done < <(find ${FOLDER} ${AUX} -name "${MASK}" -type f -size ${PARAM1^^})
}
 
########################################################################
#
# / FUNCTIONS
#
########################################################################
 
 
########################################################################
#
#  MAIN
#
########################################################################
 
 
[ ! -d ${LOGDIR} ] && mkdir -p ${LOGDIR}
 
exec 1>> ${LOGDIR}/$(basename $0 .sh)_${MYDATE}.log
exec 2>> ${LOGDIR}/$(basename $0 .sh)_${MYDATE}.err
 
 
if [ ! -f ${CONFIGFILE} ]; then
        echo "No configuration file found" 
        exit 1
fi
 
 
 
# workaround for eval "bug"
AUXDIR="/tmp/$(date +%s)"
mkdir ${AUXDIR}
cd ${AUXDIR}
#/ workaround for eval "bug"
 
while read LINE ; do
 
 
    if [[ ! "${LINE,,}" =~ ^\/[a-z0-9]{1,}\/[\.\/\_\ a-z0-9\-]{1,},[\*\.\_\ \#a-z0-9\-]{3,20},(dirbydate|dirbymin|bydate|bymin|rotate|rotatezip|zip),\+[0-9]{1,3}(|[kmg]),(|[0-9]{1,3}),(|\-[a-z0-9]{1,10})$ ]] ; then
        printf "%sskipping ${LINE}\nnot matching the config pattern, read instructions\n"
        continue
    fi
 
    FOLDER=$(echo $LINE | cut -d',' -f1)
    MASK=$(echo $LINE | cut -d',' -f2)
    ACTION=$(echo $LINE | cut -d',' -f3)
    PARAM1=$(echo $LINE | cut -d',' -f4)
    PARAM2=$(echo $LINE | cut -d',' -f5)
    PARAM3=$(echo $LINE | cut -d',' -f6)
 
    case ${ACTION,,} in
        "bydate" )
            remove_files mtime
        ;;
        "bymin" )
            remove_files mmin
        ;;
       "dirbydate" )
            remove_files mtime dir
        ;;
        "dirbymin" )
            remove_files mmin dir
        ;;
        "rotate" )
            rotate_files nozip
        ;;
        "rotatezip" )
            rotate_files zip
        ;;
        "zip" )
            printf "%s$(date +%Y%m%d%H%M%S) : Gzipping files on ${FOLDER}, with filemask ${MASK}, ${OPT} ${PARAM1}"
            [[ "${PARAM2}" ]] && AUX="-maxdepth ${PARAM2}" && printf "%s and maxdepth ${PARAM2}\n" || printf "%s\n"
            [[ ! "${PARAM3}" ]] && PARAM3="-9fv"
            find ${FOLDER} ${AUX} -name "${MASK}" -type f -mtime ${PARAM1} -exec gzip ${PARAM3} {} \;
        ;;
        * )
            usage
            exit 2
        ;;
    esac
 
done < <(cat ${CONFIGFILE} | egrep -v "^#|^$")
 
 
 
# workaround for eval "bug"
cd ${OLDPWD}
rm -fr ${AUXDIR}
#/ workaround for eval "bug"
 
########################################################################
#
# / MAIN
#
########################################################################
linux/purging_script.1381404145.txt.gz · Last modified: 2013/10/10 11:22 by dodger