added RSS feed, moved health processing to a function
This commit is contained in:
mischa
parent
38f7df88e7
commit
fcac957478
15
README.md
15
README.md
|
|
@ -10,7 +10,8 @@ uptimeatomic alerts when downtime happens and generates an html status page via
|
|||
* Minimal dependencies (curl, nc and coreutils)
|
||||
* Easy configuration and customisation
|
||||
* Tiny (~1kb) optimized result page
|
||||
* Incident history (manual)
|
||||
* Incident and maintenance history (manual)
|
||||
* RSS feed for incidents and maintenance messages
|
||||
* Crontab friendly
|
||||
|
||||
## Demo
|
||||
|
|
@ -52,13 +53,23 @@ Note: `port4` and `port6` require OpenBSD `nc` binary.
|
|||
|
||||
The default timeout is set in `uptimeatomic` but can be set in `checks.csv` per host.
|
||||
|
||||
## Incidents / Maintenance
|
||||
|
||||
The syntax of the `incidents.txt` and `pastincidents.txt` files is:
|
||||
|
||||
```
|
||||
Datetime, Type, Description
|
||||
202305231230, Incident, There is a service interuption on 2023-05-23 at 12:23 UTC
|
||||
202306031700, Maintenance, Server maintenance is scheduled on 2023-07-13 at 00:00 UTC
|
||||
|
||||
## Parameters
|
||||
|
||||
```
|
||||
./uptimeatomic -c CHECKFILE -i INCIDENTSFILE -o HTMLFILE
|
||||
./uptimeatomic -c CHECKFILE -i INCIDENTSFILE -o HTMLFILE -r RSSFILE
|
||||
Default:
|
||||
-c = checks.csv
|
||||
-i = incidents.txt
|
||||
-p = pastincidents.txt
|
||||
-o = index.html
|
||||
-r = rss file (no default)
|
||||
```
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
202306021200,Maintenance,On 2023-06-15 between 22:30-02:30 UTC our upstream provider has scheduled network maintenance
|
||||
|
|
@ -1 +1 @@
|
|||
2023-05-28 21:17 - Short network outage was observed at our upstream provider between 2023-05-28 21:17:28 UTC and 2023-05-28 21:19:17 UTC
|
||||
202305282120,Incident,Between 21:17:28 and 21:19:17 UTC on 2023-05-28 a network outage happened at our upstream provider. Resolved
|
||||
|
|
|
|||
|
|
@ -0,0 +1,4 @@
|
|||
# Ignore everything in this directory
|
||||
*
|
||||
# Except this file
|
||||
!.gitignore
|
||||
281
uptimeatomic
281
uptimeatomic
|
|
@ -1,6 +1,6 @@
|
|||
#!/bin/ksh
|
||||
#
|
||||
# Uptime Atomic v20230531
|
||||
# Uptime Atomic v20230603
|
||||
# https://git.high5.nl/uptimeatomic/
|
||||
#
|
||||
export PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin
|
||||
|
|
@ -16,9 +16,12 @@ PUSHOVER_CONF="/home/mischa/_pushover.conf"
|
|||
PUSHOVER_STATUS="status"
|
||||
RECIPIENT="alerts@high5.nl"
|
||||
REFRESH=60
|
||||
RSS_DESCRIPTION="Incidents and Maintenance Updates"
|
||||
RSS_FILE=""
|
||||
RSS_URL="https://ops.lowfive.nl"
|
||||
SENDER="uptimeatomic@high5.nl"
|
||||
TIMEOUT=10
|
||||
TITLE="Uptime Atomic"
|
||||
TITLE="Uptime Atomic - OpenBSD Amsterdam"
|
||||
USERAGENT="User-Agent: Mozilla/5.0 (OpenBSD; Intel OpenBSD 7.4; rv:109.0) Gecko/20100101 Firefox/113.0"
|
||||
WORKDIR=/home/mischa/uptimeatomic
|
||||
|
||||
|
|
@ -27,82 +30,161 @@ usage() {
|
|||
exit
|
||||
}
|
||||
|
||||
date_rss() {
|
||||
if [ -n "${1}" ]; then
|
||||
date -j '+%a, %d %b %Y %H:%M:%S %z' $(echo "$1")
|
||||
else
|
||||
date -j '+%a, %d %b %Y %H:%M:%S %z'
|
||||
fi
|
||||
}
|
||||
date_incident() {
|
||||
if [ -n "${1}" ]; then
|
||||
date -j '+%F %H:%M %Z' $(echo "$1")
|
||||
else
|
||||
date -j '+%F %H:%M %Z'
|
||||
fi
|
||||
}
|
||||
|
||||
get_element() {
|
||||
echo "${2}" | awk -v col="${1}" -F',' '{gsub(/^[ \t]+|[ \t]+$/, "", $col); print $col}'
|
||||
}
|
||||
|
||||
notify() {
|
||||
name="${1}"
|
||||
status="${2}"
|
||||
priority="${3}"
|
||||
|
||||
if [ ${priority} == "KO" ]; then
|
||||
echo "${status}." | mail -r "${TITLE} <${SENDER}>" -s "${name} DOWN" ${RECIPIENT}
|
||||
${PUSHOVER} -c ${PUSHOVER_CONF} -t "${TITLE}" -m "${name} DOWN ${status}." -p 1 >/dev/null 2>&1
|
||||
touch "${PUSHOVER_STATUS}/${name}"
|
||||
incidents() {
|
||||
_date=${1}
|
||||
_type=${2}
|
||||
_description=${3}
|
||||
[[ ${_type} == "Incident" ]] && color="failed"
|
||||
[[ ${_type} == "Maintenance" ]] && color="maint"
|
||||
echo "<p>$(date_incident ${_date}) - <b class='${color}'>${_type}</b><br />${_description}</p>" >> ${_HTMLFILE}
|
||||
if [ -n ${RSS_FILE} ]; then
|
||||
cat << EOF >> ${RSS_FILE}
|
||||
<item>
|
||||
<title>${_type}</title>
|
||||
<link>${RSS_URL}/rss.xml</link>
|
||||
<guid>${RSS_URL}/rss.xml?$(date -j '+%s' $(echo "${_date}"))</guid>
|
||||
<pubDate>$(date_rss "${_date}")</pubDate>
|
||||
<description><![CDATA[ ${_description} ]]></description>
|
||||
</item>
|
||||
EOF
|
||||
fi
|
||||
if [ ${priority} == "OK" ]; then
|
||||
_seconds=$(expr $(date +%s) - $(stat -r "${PUSHOVER_STATUS}/${name}" | awk '{print $11}'))
|
||||
}
|
||||
|
||||
notify() {
|
||||
_name="${1}"
|
||||
_status="${2}"
|
||||
_priority="${3}"
|
||||
|
||||
if [ ${_priority} == "ko" ]; then
|
||||
echo "${_status}." | mail -r "${TITLE} <${SENDER}>" -s "${_name} DOWN" ${RECIPIENT}
|
||||
${PUSHOVER} -c ${PUSHOVER_CONF} -t "${TITLE}" -m "${_name} DOWN ${_status}." -p 1 >/dev/null 2>&1
|
||||
touch "${PUSHOVER_STATUS}/${_name}"
|
||||
fi
|
||||
if [ ${_priority} == "ok" ]; then
|
||||
_seconds=$(expr $(date +%s) - $(stat -r "${PUSHOVER_STATUS}/${_name}" | awk '{print $11}'))
|
||||
_downtime=$(date -r${_seconds} -u +%H:%M:%S)
|
||||
|
||||
echo "${status} - down for ${_downtime}" | mail -r "${TITLE} <${SENDER}>" -s "${name} OK" ${RECIPIENT}
|
||||
${PUSHOVER} -c ${PUSHOVER_CONF} -t "${TITLE}" -m "${name} OK ${status} - down for ${_downtime}" >/dev/null 2>&1
|
||||
rm -rf "${PUSHOVER_STATUS}/${name}"
|
||||
echo "${_status} - down for ${_downtime}" | mail -r "${TITLE} <${SENDER}>" -s "${_name} OK" ${RECIPIENT}
|
||||
${PUSHOVER} -c ${PUSHOVER_CONF} -t "${TITLE}" -m "${_name} OK ${_status} - down for ${_downtime}" >/dev/null 2>&1
|
||||
rm -rf "${PUSHOVER_STATUS}/${_name}"
|
||||
fi
|
||||
}
|
||||
|
||||
check() {
|
||||
ctype="${1}"
|
||||
host="${2}"
|
||||
name="${3}"
|
||||
expectedcode="${4}"
|
||||
timeout="${5}"
|
||||
IPv="${ctype#(http|ping|port)}"
|
||||
[[ -n "${timeout}" ]] && TIMEOUT=${timeout}
|
||||
_ctype="${1}"
|
||||
_expectedcode="${2}"
|
||||
_name="${3}"
|
||||
_host="${4}"
|
||||
_timeout="${5}"
|
||||
IPv="${_ctype#(http|ping|port)}"
|
||||
[[ -n "${_timeout}" ]] && TIMEOUT=${_timeout}
|
||||
|
||||
case "${ctype}" in
|
||||
case "${_ctype}" in
|
||||
http*)
|
||||
statuscode="$(curl -${IPv}sSkLo /dev/null -H "${USERAGENT}" -m "${TIMEOUT}" -w "%{http_code}" "${host}" 2> "${_tmp}/ko/${name}.error")"
|
||||
if [ "${statuscode}" -ne "${expectedcode}" ]; then
|
||||
if [ -s "${_tmp}/ko/${name}.error" ]; then
|
||||
sed -e 's,curl: ([0-9]*) ,,' "${_tmp}/ko/${name}.error" > "${_tmp}/ko/${name}.status"
|
||||
statuscode="$(curl -${IPv}sSkLo /dev/null -H "${USERAGENT}" -m "${TIMEOUT}" -w "%{http_code}" "${_host}" 2> "${_TMP}/ko/${_name}.error")"
|
||||
if [ "${statuscode}" -ne "${_expectedcode}" ]; then
|
||||
if [ -s "${_TMP}/ko/${_name}.error" ]; then
|
||||
sed -e 's,curl: ([0-9]*) ,,' "${_TMP}/ko/${_name}.error" > "${_TMP}/ko/${_name}.status"
|
||||
else
|
||||
echo "Status code: ${statuscode}" > "${_tmp}/ko/${name}.status"
|
||||
echo "Status code: ${statuscode}" > "${_TMP}/ko/${_name}.status"
|
||||
fi
|
||||
else
|
||||
echo "Status code: ${statuscode}" > "${_tmp}/ok/${name}.status"
|
||||
echo "Status code: ${statuscode}" > "${_TMP}/ok/${_name}.status"
|
||||
fi
|
||||
;;
|
||||
ping*)
|
||||
ping -${IPv}w "${TIMEOUT}" -c 1 "${host}" >/dev/null 2>&1
|
||||
ping -${IPv}w "${TIMEOUT}" -c 1 "${_host}" >/dev/null 2>&1
|
||||
statuscode=$?
|
||||
if [ "${statuscode}" -ne "${expectedcode}" ]; then
|
||||
echo "Host unreachable" > "${_tmp}/ko/${name}.status"
|
||||
if [ "${statuscode}" -ne "${_expectedcode}" ]; then
|
||||
echo "Host unreachable" > "${_TMP}/ko/${_name}.status"
|
||||
else
|
||||
echo "Host reachable" > "${_tmp}/ok/${name}.status"
|
||||
echo "Host reachable" > "${_TMP}/ok/${_name}.status"
|
||||
fi
|
||||
;;
|
||||
port*)
|
||||
error="$(nc -${IPv}w "${TIMEOUT}" -zv ${host} 2>&1)"
|
||||
error="$(nc -${IPv}w "${TIMEOUT}" -zv ${_host} 2>&1)"
|
||||
statuscode=$?
|
||||
if [ "${statuscode}" -ne "${expectedcode}" ]; then
|
||||
echo "Connection refused" > "${_tmp}/ko/${name}.status"
|
||||
if [ "${statuscode}" -ne "${_expectedcode}" ]; then
|
||||
echo "Connection refused" > "${_TMP}/ko/${_name}.status"
|
||||
else
|
||||
echo "Connection succeeded" > "${_tmp}/ok/${name}.status"
|
||||
echo "Connection succeeded" > "${_TMP}/ok/${_name}.status"
|
||||
fi
|
||||
;;
|
||||
maint*)
|
||||
echo "Maintenance" > "${_tmp}/maint/${name}.status"
|
||||
echo "Maintenance" > "${_TMP}/maint/${_name}.status"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
while getopts c:i:o:h arg; do
|
||||
process_health() {
|
||||
_health=${1}
|
||||
|
||||
ls ${_TMP}/${_health}/*.status 2>/dev/null | sort -V | while read file; do
|
||||
[ -e "${file}" ] || continue
|
||||
_name="$(basename "${file%.status}")"
|
||||
_status="$(cat "${file}")"
|
||||
|
||||
if [ ${_health} == "ko" ]; then
|
||||
echo "<li>${_name} <span class='small failed'>(${_status})</span><span class='status failed'>Disrupted</span></li>" >> ${_HTMLFILE}
|
||||
if [ ! -e "${PUSHOVER_STATUS}/${_name}" ]; then
|
||||
notify "${_name}" "${status}" "${_health}"
|
||||
fi
|
||||
fi
|
||||
if [ ${_health} == "maint" ]; then
|
||||
echo "<li>${_name} <span class='status maint'>Maintenance</span></li>" >> ${_HTMLFILE}
|
||||
fi
|
||||
if [ ${_health} == "ok" ]; then
|
||||
echo "<li>${_name} <span class='status success'>Operational</span></li>" >> ${_HTMLFILE}
|
||||
if [ -e "${PUSHOVER_STATUS}/${_name}" ]; then
|
||||
notify "${_name}" "${_status}" "${_health}"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
parse_file() {
|
||||
_file=${1}
|
||||
while IFS="$(printf '\n')" read -r line; do
|
||||
_col1="$(get_element 1 "${line}")"
|
||||
_col2="$(get_element 2 "${line}")"
|
||||
_col3="$(get_element 3 "${line}")"
|
||||
_col4="$(get_element 4 "${line}")"
|
||||
_col5="$(get_element 5 "${line}")"
|
||||
if [[ ${_file} == *".csv" ]]; then
|
||||
check "${_col1}" "${_col2}" "${_col3}" "${_col4}" "${_col5}" &
|
||||
else
|
||||
incidents "${_col1}" "${_col2}" "${_col3}"
|
||||
fi
|
||||
done < "${_file}"
|
||||
wait
|
||||
}
|
||||
|
||||
while getopts c:i:o:r:h arg; do
|
||||
case ${arg} in
|
||||
c) CHECKFILE=${OPTARG};;
|
||||
i) INCIDENTSFILE=${OPTARG};;
|
||||
p) PASTINCIDENTSFILE=${OPTARG};;
|
||||
o) HTMLFILE=${OPTARG};;
|
||||
r) RSS_FILE=${OPTARG};;
|
||||
h) usage;;
|
||||
*) usage;;
|
||||
esac
|
||||
|
|
@ -114,21 +196,16 @@ if [ ! -e "${CHECKFILE}" ]; then
|
|||
exit
|
||||
fi
|
||||
|
||||
_tmp="$(mktemp -d)"
|
||||
mkdir -p "${_tmp}/ok" "${_tmp}/ko" "${_tmp}/maint" || exit 1
|
||||
_htmlfile="${_tmp}/${HTMLFILE}"
|
||||
_TMP="$(mktemp -d)"
|
||||
mkdir -p "${_TMP}/ok" "${_TMP}/ko" "${_TMP}/maint" || exit 1
|
||||
_HTMLFILE="${_TMP}/${HTMLFILE}"
|
||||
|
||||
while IFS="$(printf '\n')" read -r line; do
|
||||
ctype="$(get_element 1 "${line}")"
|
||||
code="$(get_element 2 "${line}")"
|
||||
name="$(get_element 3 "${line}")"
|
||||
host="$(get_element 4 "${line}")"
|
||||
timeout="$(get_element 5 "${line}")"
|
||||
check "${ctype}" "${host}" "${name}" "${code}" "${timeout}" &
|
||||
done < "${CHECKFILE}"
|
||||
wait
|
||||
parse_file "${CHECKFILE}"
|
||||
|
||||
cat << EOF >> ${_htmlfile}
|
||||
#
|
||||
# HTML
|
||||
#
|
||||
cat << EOF > ${_HTMLFILE}
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
|
|
@ -157,62 +234,74 @@ li { list-style: none; margin-bottom: 2px; padding: 5px; border-bottom: 1px soli
|
|||
<div class='container'>
|
||||
<h1>${HEADER}</h1>
|
||||
EOF
|
||||
_outage_count="$(find "${_tmp}/ko" -mindepth 1 | grep -c 'status$')"
|
||||
_maint_count="$(find "${_tmp}/maint" -mindepth 1 | grep -c 'status$')"
|
||||
|
||||
#
|
||||
# RSS
|
||||
#
|
||||
if [ -n ${RSS_FILE} ]; then
|
||||
cat << EOF > ${RSS_FILE}
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
|
||||
<channel>
|
||||
<atom:link href="${URL}/rss.xml" rel="self" type="application/rss+xml" />
|
||||
<title>${TITLE}</title>
|
||||
<description>${RSS_DESCRIPTION}</description>
|
||||
<link>${RSS_URL}</link>
|
||||
<lastBuildDate>$(date_rss)</lastBuildDate>
|
||||
EOF
|
||||
fi
|
||||
|
||||
_outage_count="$(find "${_TMP}/ko" -mindepth 1 | grep -c 'status$')"
|
||||
_maint_count="$(find "${_TMP}/maint" -mindepth 1 | grep -c 'status$')"
|
||||
if [ "${_outage_count}" -ne 0 ]; then
|
||||
echo "<ul><li class='panel failed-bg'>${_outage_count} Outage(s)</li></ul>" >> ${_htmlfile}
|
||||
echo "<ul><li class='panel failed-bg'>${_outage_count} Outage(s)</li></ul>" >> ${_HTMLFILE}
|
||||
fi
|
||||
if [ "${_maint_count}" -ne 0 ]; then
|
||||
echo "<ul><li class='panel maint-bg'>${_maint_count} Maintenance</li></ul>" >> ${_htmlfile}
|
||||
echo "<ul><li class='panel maint-bg'>${_maint_count} Maintenance</li></ul>" >> ${_HTMLFILE}
|
||||
fi
|
||||
if [[ "${_outage_count}" -eq 0 && "${_maint_count}" -eq 0 ]]; then
|
||||
echo "<ul><li class='panel success-bg'>All Systems Operational</li></ul>" >> ${_htmlfile}
|
||||
echo "<ul><li class='panel success-bg'>All Systems Operational</li></ul>" >> ${_HTMLFILE}
|
||||
fi
|
||||
|
||||
if [ -s "${INCIDENTSFILE}" ]; then
|
||||
echo '<h1>Incidents</h1>' >> ${_htmlfile}
|
||||
sed 's|^\(.*\)$|<p>\1</p>|' "${INCIDENTSFILE}" >> ${_htmlfile}
|
||||
echo '<h2>Incidents / Maintenance</h2>' >> ${_HTMLFILE}
|
||||
parse_file "${INCIDENTSFILE}"
|
||||
fi
|
||||
cat << EOF >> ${_htmlfile}
|
||||
<h1>Services</h1>
|
||||
<ul>
|
||||
EOF
|
||||
ls ${_tmp}/ko/*.status 2>/dev/null | sort -V | while read file; do
|
||||
[ -e "${file}" ] || continue
|
||||
name="$(basename "${file%.status}")"
|
||||
status="$(cat "${file}")"
|
||||
echo "<li>${name} <span class='small failed'>(${status})</span><span class='status failed'>Disrupted</span></li>" >> ${_htmlfile}
|
||||
if [ ! -e "${PUSHOVER_STATUS}/${name}" ]; then
|
||||
notify "${name}" "${status}" "KO"
|
||||
fi
|
||||
done
|
||||
ls ${_tmp}/maint/*.status 2>/dev/null | sort -V | while read file; do
|
||||
[ -e "${file}" ] || continue
|
||||
name="$(basename "${file%.status}")"
|
||||
echo "<li>${name} <span class='status maint'>Maintenance</span></li>" >> ${_htmlfile}
|
||||
done
|
||||
ls ${_tmp}/ok/*.status 2>/dev/null | sort -V | while read file; do
|
||||
[ -e "${file}" ] || continue
|
||||
name="$(basename "${file%.status}")"
|
||||
status="$(cat "${file}")"
|
||||
echo "<li>${name} <span class='status success'>Operational</span></li>" >> ${_htmlfile}
|
||||
if [ -e "${PUSHOVER_STATUS}/${name}" ]; then
|
||||
notify "${name}" "${status}" "OK"
|
||||
fi
|
||||
done
|
||||
cat << EOF >> ${_htmlfile}
|
||||
</ul>
|
||||
<p class=small>Last check: $(date +%FT%T%z)</p>
|
||||
EOF
|
||||
|
||||
echo "<h1>Services</h1>" >> ${_HTMLFILE}
|
||||
echo "<ul>" >> ${_HTMLFILE}
|
||||
|
||||
process_health "ko"
|
||||
process_health "maint"
|
||||
process_health "ok"
|
||||
|
||||
echo "</ul>" >> ${_HTMLFILE}
|
||||
echo "<p class=small>Last check: $(date '+%FT%T %Z')</p>" >> ${_HTMLFILE}
|
||||
|
||||
if [ -s "${PASTINCIDENTSFILE}" ]; then
|
||||
echo '<h1>Past Incidents</h1>' >> ${_htmlfile}
|
||||
sed 's|^\(.*\)$|<p>\1</p>|' "${PASTINCIDENTSFILE}" >> ${_htmlfile}
|
||||
echo '<h3>Past Incidents / Maintenance</h3>' >> ${_HTMLFILE}
|
||||
parse_file "${PASTINCIDENTSFILE}"
|
||||
fi
|
||||
cat << EOF >> ${_htmlfile}
|
||||
|
||||
#
|
||||
# END HTML
|
||||
#
|
||||
cat << EOF >> ${_HTMLFILE}
|
||||
<p class=small><a href="https://git.high5.nl/uptimeatomic/">Uptime Atomic</a> loosely based on <a href="https://github.com/bderenzo/tinystatus">Tinystatus</a></p>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
EOF
|
||||
|
||||
cp ${_htmlfile} ${HTMLDIR}/${HTMLFILE}
|
||||
rm -r "${_tmp}" 2>/dev/null
|
||||
#
|
||||
# END RSS
|
||||
#
|
||||
if [ -n ${RSS_FILE} ]; then
|
||||
cat << EOF >> ${RSS_FILE}
|
||||
</channel>
|
||||
</rss>
|
||||
EOF
|
||||
fi
|
||||
|
||||
cp ${_HTMLFILE} ${HTMLDIR}/${HTMLFILE}
|
||||
rm -r "${_TMP}" 2>/dev/null
|
||||
|
|
|
|||
Loading…
Reference in New Issue