系統環境：
SUSE Linux Enterprise Server 10 SP1 (x86_64)
問題背景：
由於線上系統環境下的crontab內容比較多，在進行日常crontab任務調度時，經常會異常掛掉而影響業務的正常使用，因此結合C和Shell寫了一個簡單的對crontab進行健康檢測的功能。
處理思路：
修改syslog的配置參數，把crontab調度日志單獨抽取出來，同時在crontab項裡添加檢測標記，通過後台守護進程定期檢測狀態標記來判斷當前crontab調度是否正常，同時為了避免日志文件過大而影響性能，會定期對日志文件做切割和清理處理。
#--------------------------------------------------------------------------------------------------------------------------------------------
1、相關目錄創建
# mkdir -p /data/logs/crontab
# mkdir -p /data/scripts
# mkdir -p /data/backup/crontab
#--------------------------------------------------------------------------------------------------------------------------------------------
2、crontab健康檢測C代碼
# cd /data/scripts
# vim check_cron_process.h
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26#ifndef __CHECK_CRON_PROCESS_H__
#define __CHECK_CRON_PROCESS_H__
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#define BUFFSIZE1 1024
#define BUFFSIZE2 32
#define LOCKFILE "/var/run/check_cron_process.pid"
#define LOGFILE "/var/log/check_cron_process.log"
#define LOCKMODE (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)
#define SYSCMD1 "ps aux | grep -w cron | grep -v grep"
#define SYSCMD2 "ps aux | grep -w cron | grep -v grep | grep defunct"
#define SYSCMD3 "tail -6 /data/logs/crontab/cron.log | grep '(root) CMD (cd'"
#define SYSCMD4 "killall -9 cron >/dev/null 2>&1"
#define SYSCMD5 "/sbin/service cron start >/dev/null 2>&1"
void already_running(void);
void init_daemon(void);
int run_system_cmd(const char *syscmd);
#endif
# vim check_cron_process.c
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/param.h>
#include <sys/stat.h>
#include <syslog.h>
#include <fcntl.h>
#include <errno.h>
#include <time.h>
#include "check_cron_process.h"
static char buffer[BUFFSIZE1] = {0};
static char datetime[BUFFSIZE2] = {0};
/* 獲得當前系統時間 */
int get_curr_date(char *strtime, unsigned int ustrlen)
{
struct tm *pt = NULL;
time_t timer;
if (!strtime) {
return -1;
}
time(&timer);
strtime[0] = '\0';
pt = localtime(&timer);
if (!pt) {
return -1;
}
memset(strtime, 0, ustrlen);
sprintf(strtime, "%04d-%02d-%02d-%02d:%02d:%02d",
pt->tm_year + 1900, pt->tm_mon + 1, pt->tm_mday, pt->tm_hour, pt->tm_min, pt->tm_sec);
return 0;
}
/* 將信息寫入日志文件 */
int writelog(const char *pLoginfo)
{
FILE *fp = NULL;
unsigned int ustrlen = 0;
if (pLoginfo == NULL) {
return -1;
}
ustrlen = strlen(pLoginfo);
if (ustrlen > 256) {
return -1;
}
if ((fp = fopen(LOGFILE, "a+")) == NULL) {
return -1;
}
memset(datetime, 0, BUFFSIZE2);
get_curr_date(datetime, BUFFSIZE2);
fprintf(fp, "%s %s", datetime, pLoginfo);
fclose(fp);
return 0;
}
int LockFile(int fd)
{
struct flock fl;
fl.l_type = F_WRLCK;
fl.l_start = 0;
fl.l_whence = SEEK_SET;
fl.l_len = 0;
return (fcntl(fd, F_SETLK, &fl));
}
/* 只允許一個副本運行 */
void already_running(void)
{
int fd = -1;
char buf[16] = {0};
fd = open(LOCKFILE, O_RDWR | O_CREAT, LOCKMODE);
if (fd < 0) {
syslog(LOG_ERR, "can't open %s: %s", LOCKFILE, strerror(errno));
exit(1);
}
if (LockFile(fd) < 0) {
if (errno == EACCES || errno == EAGAIN) {
close(fd);
exit(1);
}
syslog(LOG_ERR, "can't lock %s: %s", LOCKFILE, strerror(errno));
exit(1);
}
ftruncate(fd, 0);
sprintf(buf, "%d", getpid());
write(fd, buf, strlen(buf));
close(fd);
}
/* 作為守護進程運行 */
void init_daemon(void)
{
int pid = -1;
if ((pid = fork())) {
exit(0);
} else if (pid < 0) {
exit(1);
}
setsid();
if ((pid = fork())) {
exit(0);
} else if (pid < 0) {
exit(1);
}
chdir("/tmp");
umask(0);
return;
}
/* 運行系統命令 */
int run_system_cmd(const char *syscmd)
{
FILE *fp = NULL;
if (syscmd == NULL) {
return -1;
}
memset(buffer, 0, BUFFSIZE1);
snprintf(buffer, BUFFSIZE1, syscmd);
fp = popen(buffer, "r");
if (!fp) {
return 0;
}
memset(buffer, 0, BUFFSIZE1);
if (!fgets(buffer, BUFFSIZE1, fp)) {
pclose(fp);
return 0;
}
if (!strncasecmp(buffer, "", BUFFSIZE1)) {
pclose(fp);
return 0;
}
pclose(fp);
return 1;
}
int main(int argc, char *argv[])
{
int ret = 0;
init_daemon();
already_running();
openlog(NULL, LOG_CONS | LOG_PID, LOG_LOCAL1);
while(1) {
/* 1.檢查cron進程是否已經運行 */
ret = run_system_cmd(SYSCMD1);
if (!ret) {
writelog("The cron process is not running, now start it! \n");
sleep(1);
system(SYSCMD5);
goto CHECK_CRON;
}
/* 2.如果已經運行，查看是否有僵屍進程 */
ret = run_system_cmd(SYSCMD2);
if (ret) {
writelog("The cron process is defunct, now restart it! \n");
sleep(1);
system(SYSCMD4);
sleep(1);
system(SYSCMD5);
goto CHECK_CRON;
}
/* 3.如果進程運行正常，檢查任務是否正常調度 */
ret = run_system_cmd(SYSCMD3);
if (!ret) {
writelog("The cron work is down, now restart it! \n");
sleep(1);
system(SYSCMD4);
sleep(1);
system(SYSCMD5);
goto CHECK_CRON;
}
/* 4.crontab進程運行正常 */
writelog("The cron process is ok! \n");
CHECK_CRON:
/* 休眠5分鐘後，繼續檢測 */
sleep(300);
}
closelog();
return 0;
}
# vim Makefile
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19CC = gcc
CXX = g++
BINARY = check_cron_process
OBJS = check_cron_process.o
CFLAGS += -I/usr/include -I/usr/local/include -Wall -Wno-unused-variable
LDFLAGS = -static -O2
all:$(BINARY)
$(BINARY):$(OBJS)
$(CC) $(LDFLAGS) -o $(BINARY) $(OBJS)
$(OBJS):%.o:%.c
$(CC) $(CFLAGS) -c $^ -o $@
clean:
rm -f $(BINARY) $(OBJS)
# make
# /data/scripts/check_cron_process
#--------------------------------------------------------------------------------------------------------------------------------------------
3、crontab任務調度日志配置【新增】
# vim /etc/syslog.conf
1 2 3## check_crontab_start.tag.1
cron.* /data/logs/crontab/cron.log
## check_crontab_end.tag.1
# vim /etc/syslog-ng/syslog-ng.conf
1 2 3 4 5 6 7 8 9 10 11## check_crontab_start.tag.1
destination dst_cron {
file("/data/logs/crontab/cron.log");
};
log {
source(src);
filter(f_cron);
destination(dst_cron);
};
## check_crontab_end.tag.1
# /sbin/service syslog restart
#--------------------------------------------------------------------------------------------------------------------------------------------
4、crontab任務調度日志處理
(1)、日志切割與清理
# vim /data/scripts/cut_cron_log.sh
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17#!/bin/bash
PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin:/usr/local/sbin
## MY CRON LOG PATH
LOGPATH="/data/logs/crontab"
retval=`ps aux | grep sbin/cron | grep -v grep | wc -l`
if [ ${retval} -eq 0 ]; then
echo "The cron process is not running ^_^"
exit 1
fi
## cut crontab's log
mv ${LOGPATH}/cron.log ${LOGPATH}/cron_$(date -d "yesterday" +"%Y-%m-%d").log
/sbin/service syslog restart
## clear 10 days ago's crontab logs
rm -f ${LOGPATH}/cron_$(date -d "10 days ago" +"%Y-%m-%d").log
(2)、crontab信息備份
# vim /data/scripts/backup_crontab.sh
1 2 3 4 5 6 7 8 9 10 11 12#!/bin/bash
PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin
CRONTAB_BACKUP_DIR="/data/backup/crontab"
#備份crontab內容
mkdir -p ${CRONTAB_BACKUP_DIR}
crontab -uroot -l > ${CRONTAB_BACKUP_DIR}/crontab_`date +%F`
#清理10前的備份
CRONBAK=crontab_$(date -d "10 days ago" +"%Y-%m-%d")
find ${CRONTAB_BACKUP_DIR} -type f -name ${CRONBAK} -exec rm -f {} \;
(3)、crontab垃圾頭信息清理
# vim /data/scripts/clean_crontab_trash.sh
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15#!/bin/bash
PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin:/usr/local/sbin
## The crontab's spool file
CRONFILE="/var/spool/cron/tabs/root"
sed -i '/# DO NOT EDIT THIS FILE/d' ${CRONFILE}
sed -i '/# (\/data\/crontab.tmp/d' ${CRONFILE}
sed -i '/# (\/tmp\/crontab/d' ${CRONFILE}
sed -i '/# (Cron version/d' ${CRONFILE}
sed -i '/# (- installed on/d' ${CRONFILE}
sed -i '/# (\/usr\/local\/agenttools/d' ${CRONFILE}
sed -i '/# (\/tmp\/cron.tmp/d' ${CRONFILE}
sed -i '/# (tmp2 installed/d' ${CRONFILE}
sed -i '/# (crontab.tmp/d' ${CRONFILE}
sed -i '/# (\/data\/crontab_/d' ${CRONFILE}
(4)、crontab設置
# crontab -e
## crontab日志切割��清理
00 00 * * * /data/scripts/cut_cron_log.sh >/dev/null 2>&1
## 運行狀況檢測標記
*/1 * * * * cd /usr/local; echo >/dev/null 2>&1
## crontab信息備份
30 08 * * * /data/scripts/backup_crontab.sh >/dev/null 2>&1
## crontab垃圾頭信息清理
*/30 * * * * /data/scripts/clean_crontab_trash.sh >/dev/null 2>&1
Linux中利用crontab創建計劃任務 http://www.linuxidc.com//Linux/2013-06/86401.htm
Linux中用crontab例行工作安排 http://www.linuxidc.com//Linux/2013-06/85441.htm
Linux crontab不執行問題排查 http://www.linuxidc.com//Linux/2013-06/85432.htm
Ubuntu使用crontab定時任務 http://www.linuxidc.com//Linux/2013-05/84770.htm
Linux計劃任務(at batch crontab anacron) http://www.linuxidc.com//Linux/2013-03/81584.htm
Linux任務計劃 (at,crontab) http://www.linuxidc.com/Linux/2015-09/122970.htm
