added RSS feed, moved health processing to a function

master
mischa 2023-06-03 15:09:17 +00:00
parent 38f7df88e7
commit fcac957478
5 changed files with 204 additions and 99 deletions

View File

@ -10,7 +10,8 @@ uptimeatomic alerts when downtime happens and generates an html status page via
* Minimal dependencies (curl, nc and coreutils)
* Easy configuration and customisation
* Tiny (~1kb) optimized result page
* Incident history (manual)
* Incident and maintenance history (manual)
* RSS feed for incidents and maintenance messages
* Crontab friendly
## Demo
@ -52,13 +53,23 @@ Note: `port4` and `port6` require OpenBSD `nc` binary.
The default timeout is set in `uptimeatomic` but can be set in `checks.csv` per host.
## Incidents / Maintenance
The syntax of the `incidents.txt` and `pastincidents.txt` files is:
```
Datetime, Type, Description
202305231230, Incident, There is a service interuption on 2023-05-23 at 12:23 UTC
202306031700, Maintenance, Server maintenance is scheduled on 2023-07-13 at 00:00 UTC
## Parameters
```
./uptimeatomic -c CHECKFILE -i INCIDENTSFILE -o HTMLFILE
./uptimeatomic -c CHECKFILE -i INCIDENTSFILE -o HTMLFILE -r RSSFILE
Default:
-c = checks.csv
-i = incidents.txt
-p = pastincidents.txt
-o = index.html
-r = rss file (no default)
```

View File

@ -0,0 +1 @@
202306021200,Maintenance,On 2023-06-15 between 22:30-02:30 UTC our upstream provider has scheduled network maintenance

View File

@ -1 +1 @@
2023-05-28 21:17 - Short network outage was observed at our upstream provider between 2023-05-28 21:17:28 UTC and 2023-05-28 21:19:17 UTC
202305282120,Incident,Between 21:17:28 and 21:19:17 UTC on 2023-05-28 a network outage happened at our upstream provider. Resolved

4
status/.gitignore vendored Normal file
View File

@ -0,0 +1,4 @@
# Ignore everything in this directory
*
# Except this file
!.gitignore

View File

@ -1,6 +1,6 @@
#!/bin/ksh
#
# Uptime Atomic v20230531
# Uptime Atomic v20230603
# https://git.high5.nl/uptimeatomic/
#
export PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin
@ -16,9 +16,12 @@ PUSHOVER_CONF="/home/mischa/_pushover.conf"
PUSHOVER_STATUS="status"
RECIPIENT="alerts@high5.nl"
REFRESH=60
RSS_DESCRIPTION="Incidents and Maintenance Updates"
RSS_FILE=""
RSS_URL="https://ops.lowfive.nl"
SENDER="uptimeatomic@high5.nl"
TIMEOUT=10
TITLE="Uptime Atomic"
TITLE="Uptime Atomic - OpenBSD Amsterdam"
USERAGENT="User-Agent: Mozilla/5.0 (OpenBSD; Intel OpenBSD 7.4; rv:109.0) Gecko/20100101 Firefox/113.0"
WORKDIR=/home/mischa/uptimeatomic
@ -27,82 +30,161 @@ usage() {
exit
}
date_rss() {
if [ -n "${1}" ]; then
date -j '+%a, %d %b %Y %H:%M:%S %z' $(echo "$1")
else
date -j '+%a, %d %b %Y %H:%M:%S %z'
fi
}
date_incident() {
if [ -n "${1}" ]; then
date -j '+%F %H:%M %Z' $(echo "$1")
else
date -j '+%F %H:%M %Z'
fi
}
get_element() {
echo "${2}" | awk -v col="${1}" -F',' '{gsub(/^[ \t]+|[ \t]+$/, "", $col); print $col}'
}
notify() {
name="${1}"
status="${2}"
priority="${3}"
if [ ${priority} == "KO" ]; then
echo "${status}." | mail -r "${TITLE} <${SENDER}>" -s "${name} DOWN" ${RECIPIENT}
${PUSHOVER} -c ${PUSHOVER_CONF} -t "${TITLE}" -m "${name} DOWN ${status}." -p 1 >/dev/null 2>&1
touch "${PUSHOVER_STATUS}/${name}"
incidents() {
_date=${1}
_type=${2}
_description=${3}
[[ ${_type} == "Incident" ]] && color="failed"
[[ ${_type} == "Maintenance" ]] && color="maint"
echo "<p>$(date_incident ${_date}) - <b class='${color}'>${_type}</b><br />${_description}</p>" >> ${_HTMLFILE}
if [ -n ${RSS_FILE} ]; then
cat << EOF >> ${RSS_FILE}
<item>
<title>${_type}</title>
<link>${RSS_URL}/rss.xml</link>
<guid>${RSS_URL}/rss.xml?$(date -j '+%s' $(echo "${_date}"))</guid>
<pubDate>$(date_rss "${_date}")</pubDate>
<description><![CDATA[ ${_description} ]]></description>
</item>
EOF
fi
if [ ${priority} == "OK" ]; then
_seconds=$(expr $(date +%s) - $(stat -r "${PUSHOVER_STATUS}/${name}" | awk '{print $11}'))
}
notify() {
_name="${1}"
_status="${2}"
_priority="${3}"
if [ ${_priority} == "ko" ]; then
echo "${_status}." | mail -r "${TITLE} <${SENDER}>" -s "${_name} DOWN" ${RECIPIENT}
${PUSHOVER} -c ${PUSHOVER_CONF} -t "${TITLE}" -m "${_name} DOWN ${_status}." -p 1 >/dev/null 2>&1
touch "${PUSHOVER_STATUS}/${_name}"
fi
if [ ${_priority} == "ok" ]; then
_seconds=$(expr $(date +%s) - $(stat -r "${PUSHOVER_STATUS}/${_name}" | awk '{print $11}'))
_downtime=$(date -r${_seconds} -u +%H:%M:%S)
echo "${status} - down for ${_downtime}" | mail -r "${TITLE} <${SENDER}>" -s "${name} OK" ${RECIPIENT}
${PUSHOVER} -c ${PUSHOVER_CONF} -t "${TITLE}" -m "${name} OK ${status} - down for ${_downtime}" >/dev/null 2>&1
rm -rf "${PUSHOVER_STATUS}/${name}"
echo "${_status} - down for ${_downtime}" | mail -r "${TITLE} <${SENDER}>" -s "${_name} OK" ${RECIPIENT}
${PUSHOVER} -c ${PUSHOVER_CONF} -t "${TITLE}" -m "${_name} OK ${_status} - down for ${_downtime}" >/dev/null 2>&1
rm -rf "${PUSHOVER_STATUS}/${_name}"
fi
}
check() {
ctype="${1}"
host="${2}"
name="${3}"
expectedcode="${4}"
timeout="${5}"
IPv="${ctype#(http|ping|port)}"
[[ -n "${timeout}" ]] && TIMEOUT=${timeout}
_ctype="${1}"
_expectedcode="${2}"
_name="${3}"
_host="${4}"
_timeout="${5}"
IPv="${_ctype#(http|ping|port)}"
[[ -n "${_timeout}" ]] && TIMEOUT=${_timeout}
case "${ctype}" in
case "${_ctype}" in
http*)
statuscode="$(curl -${IPv}sSkLo /dev/null -H "${USERAGENT}" -m "${TIMEOUT}" -w "%{http_code}" "${host}" 2> "${_tmp}/ko/${name}.error")"
if [ "${statuscode}" -ne "${expectedcode}" ]; then
if [ -s "${_tmp}/ko/${name}.error" ]; then
sed -e 's,curl: ([0-9]*) ,,' "${_tmp}/ko/${name}.error" > "${_tmp}/ko/${name}.status"
statuscode="$(curl -${IPv}sSkLo /dev/null -H "${USERAGENT}" -m "${TIMEOUT}" -w "%{http_code}" "${_host}" 2> "${_TMP}/ko/${_name}.error")"
if [ "${statuscode}" -ne "${_expectedcode}" ]; then
if [ -s "${_TMP}/ko/${_name}.error" ]; then
sed -e 's,curl: ([0-9]*) ,,' "${_TMP}/ko/${_name}.error" > "${_TMP}/ko/${_name}.status"
else
echo "Status code: ${statuscode}" > "${_tmp}/ko/${name}.status"
echo "Status code: ${statuscode}" > "${_TMP}/ko/${_name}.status"
fi
else
echo "Status code: ${statuscode}" > "${_tmp}/ok/${name}.status"
echo "Status code: ${statuscode}" > "${_TMP}/ok/${_name}.status"
fi
;;
ping*)
ping -${IPv}w "${TIMEOUT}" -c 1 "${host}" >/dev/null 2>&1
ping -${IPv}w "${TIMEOUT}" -c 1 "${_host}" >/dev/null 2>&1
statuscode=$?
if [ "${statuscode}" -ne "${expectedcode}" ]; then
echo "Host unreachable" > "${_tmp}/ko/${name}.status"
if [ "${statuscode}" -ne "${_expectedcode}" ]; then
echo "Host unreachable" > "${_TMP}/ko/${_name}.status"
else
echo "Host reachable" > "${_tmp}/ok/${name}.status"
echo "Host reachable" > "${_TMP}/ok/${_name}.status"
fi
;;
port*)
error="$(nc -${IPv}w "${TIMEOUT}" -zv ${host} 2>&1)"
error="$(nc -${IPv}w "${TIMEOUT}" -zv ${_host} 2>&1)"
statuscode=$?
if [ "${statuscode}" -ne "${expectedcode}" ]; then
echo "Connection refused" > "${_tmp}/ko/${name}.status"
if [ "${statuscode}" -ne "${_expectedcode}" ]; then
echo "Connection refused" > "${_TMP}/ko/${_name}.status"
else
echo "Connection succeeded" > "${_tmp}/ok/${name}.status"
echo "Connection succeeded" > "${_TMP}/ok/${_name}.status"
fi
;;
maint*)
echo "Maintenance" > "${_tmp}/maint/${name}.status"
echo "Maintenance" > "${_TMP}/maint/${_name}.status"
;;
esac
}
while getopts c:i:o:h arg; do
process_health() {
_health=${1}
ls ${_TMP}/${_health}/*.status 2>/dev/null | sort -V | while read file; do
[ -e "${file}" ] || continue
_name="$(basename "${file%.status}")"
_status="$(cat "${file}")"
if [ ${_health} == "ko" ]; then
echo "<li>${_name} <span class='small failed'>(${_status})</span><span class='status failed'>Disrupted</span></li>" >> ${_HTMLFILE}
if [ ! -e "${PUSHOVER_STATUS}/${_name}" ]; then
notify "${_name}" "${status}" "${_health}"
fi
fi
if [ ${_health} == "maint" ]; then
echo "<li>${_name} <span class='status maint'>Maintenance</span></li>" >> ${_HTMLFILE}
fi
if [ ${_health} == "ok" ]; then
echo "<li>${_name} <span class='status success'>Operational</span></li>" >> ${_HTMLFILE}
if [ -e "${PUSHOVER_STATUS}/${_name}" ]; then
notify "${_name}" "${_status}" "${_health}"
fi
fi
done
}
parse_file() {
_file=${1}
while IFS="$(printf '\n')" read -r line; do
_col1="$(get_element 1 "${line}")"
_col2="$(get_element 2 "${line}")"
_col3="$(get_element 3 "${line}")"
_col4="$(get_element 4 "${line}")"
_col5="$(get_element 5 "${line}")"
if [[ ${_file} == *".csv" ]]; then
check "${_col1}" "${_col2}" "${_col3}" "${_col4}" "${_col5}" &
else
incidents "${_col1}" "${_col2}" "${_col3}"
fi
done < "${_file}"
wait
}
while getopts c:i:o:r:h arg; do
case ${arg} in
c) CHECKFILE=${OPTARG};;
i) INCIDENTSFILE=${OPTARG};;
p) PASTINCIDENTSFILE=${OPTARG};;
o) HTMLFILE=${OPTARG};;
r) RSS_FILE=${OPTARG};;
h) usage;;
*) usage;;
esac
@ -114,21 +196,16 @@ if [ ! -e "${CHECKFILE}" ]; then
exit
fi
_tmp="$(mktemp -d)"
mkdir -p "${_tmp}/ok" "${_tmp}/ko" "${_tmp}/maint" || exit 1
_htmlfile="${_tmp}/${HTMLFILE}"
_TMP="$(mktemp -d)"
mkdir -p "${_TMP}/ok" "${_TMP}/ko" "${_TMP}/maint" || exit 1
_HTMLFILE="${_TMP}/${HTMLFILE}"
while IFS="$(printf '\n')" read -r line; do
ctype="$(get_element 1 "${line}")"
code="$(get_element 2 "${line}")"
name="$(get_element 3 "${line}")"
host="$(get_element 4 "${line}")"
timeout="$(get_element 5 "${line}")"
check "${ctype}" "${host}" "${name}" "${code}" "${timeout}" &
done < "${CHECKFILE}"
wait
parse_file "${CHECKFILE}"
cat << EOF >> ${_htmlfile}
#
# HTML
#
cat << EOF > ${_HTMLFILE}
<!DOCTYPE html>
<html lang="en">
<head>
@ -157,62 +234,74 @@ li { list-style: none; margin-bottom: 2px; padding: 5px; border-bottom: 1px soli
<div class='container'>
<h1>${HEADER}</h1>
EOF
_outage_count="$(find "${_tmp}/ko" -mindepth 1 | grep -c 'status$')"
_maint_count="$(find "${_tmp}/maint" -mindepth 1 | grep -c 'status$')"
#
# RSS
#
if [ -n ${RSS_FILE} ]; then
cat << EOF > ${RSS_FILE}
<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
<channel>
<atom:link href="${URL}/rss.xml" rel="self" type="application/rss+xml" />
<title>${TITLE}</title>
<description>${RSS_DESCRIPTION}</description>
<link>${RSS_URL}</link>
<lastBuildDate>$(date_rss)</lastBuildDate>
EOF
fi
_outage_count="$(find "${_TMP}/ko" -mindepth 1 | grep -c 'status$')"
_maint_count="$(find "${_TMP}/maint" -mindepth 1 | grep -c 'status$')"
if [ "${_outage_count}" -ne 0 ]; then
echo "<ul><li class='panel failed-bg'>${_outage_count} Outage(s)</li></ul>" >> ${_htmlfile}
echo "<ul><li class='panel failed-bg'>${_outage_count} Outage(s)</li></ul>" >> ${_HTMLFILE}
fi
if [ "${_maint_count}" -ne 0 ]; then
echo "<ul><li class='panel maint-bg'>${_maint_count} Maintenance</li></ul>" >> ${_htmlfile}
echo "<ul><li class='panel maint-bg'>${_maint_count} Maintenance</li></ul>" >> ${_HTMLFILE}
fi
if [[ "${_outage_count}" -eq 0 && "${_maint_count}" -eq 0 ]]; then
echo "<ul><li class='panel success-bg'>All Systems Operational</li></ul>" >> ${_htmlfile}
echo "<ul><li class='panel success-bg'>All Systems Operational</li></ul>" >> ${_HTMLFILE}
fi
if [ -s "${INCIDENTSFILE}" ]; then
echo '<h1>Incidents</h1>' >> ${_htmlfile}
sed 's|^\(.*\)$|<p>\1</p>|' "${INCIDENTSFILE}" >> ${_htmlfile}
echo '<h2>Incidents / Maintenance</h2>' >> ${_HTMLFILE}
parse_file "${INCIDENTSFILE}"
fi
cat << EOF >> ${_htmlfile}
<h1>Services</h1>
<ul>
EOF
ls ${_tmp}/ko/*.status 2>/dev/null | sort -V | while read file; do
[ -e "${file}" ] || continue
name="$(basename "${file%.status}")"
status="$(cat "${file}")"
echo "<li>${name} <span class='small failed'>(${status})</span><span class='status failed'>Disrupted</span></li>" >> ${_htmlfile}
if [ ! -e "${PUSHOVER_STATUS}/${name}" ]; then
notify "${name}" "${status}" "KO"
fi
done
ls ${_tmp}/maint/*.status 2>/dev/null | sort -V | while read file; do
[ -e "${file}" ] || continue
name="$(basename "${file%.status}")"
echo "<li>${name} <span class='status maint'>Maintenance</span></li>" >> ${_htmlfile}
done
ls ${_tmp}/ok/*.status 2>/dev/null | sort -V | while read file; do
[ -e "${file}" ] || continue
name="$(basename "${file%.status}")"
status="$(cat "${file}")"
echo "<li>${name} <span class='status success'>Operational</span></li>" >> ${_htmlfile}
if [ -e "${PUSHOVER_STATUS}/${name}" ]; then
notify "${name}" "${status}" "OK"
fi
done
cat << EOF >> ${_htmlfile}
</ul>
<p class=small>Last check: $(date +%FT%T%z)</p>
EOF
echo "<h1>Services</h1>" >> ${_HTMLFILE}
echo "<ul>" >> ${_HTMLFILE}
process_health "ko"
process_health "maint"
process_health "ok"
echo "</ul>" >> ${_HTMLFILE}
echo "<p class=small>Last check: $(date '+%FT%T %Z')</p>" >> ${_HTMLFILE}
if [ -s "${PASTINCIDENTSFILE}" ]; then
echo '<h1>Past Incidents</h1>' >> ${_htmlfile}
sed 's|^\(.*\)$|<p>\1</p>|' "${PASTINCIDENTSFILE}" >> ${_htmlfile}
echo '<h3>Past Incidents / Maintenance</h3>' >> ${_HTMLFILE}
parse_file "${PASTINCIDENTSFILE}"
fi
cat << EOF >> ${_htmlfile}
#
# END HTML
#
cat << EOF >> ${_HTMLFILE}
<p class=small><a href="https://git.high5.nl/uptimeatomic/">Uptime Atomic</a> loosely based on <a href="https://github.com/bderenzo/tinystatus">Tinystatus</a></p>
</div>
</body>
</html>
EOF
cp ${_htmlfile} ${HTMLDIR}/${HTMLFILE}
rm -r "${_tmp}" 2>/dev/null
#
# END RSS
#
if [ -n ${RSS_FILE} ]; then
cat << EOF >> ${RSS_FILE}
</channel>
</rss>
EOF
fi
cp ${_HTMLFILE} ${HTMLDIR}/${HTMLFILE}
rm -r "${_TMP}" 2>/dev/null