From: Kristian Kræmmer Nielsen <jkkn@jkkn.dk>
Date: Mon, 13 Feb 2017 17:59:40 +0000 (+0100)
Subject: Initial distributed version
X-Git-Url: https://git.jkkn.net/?a=commitdiff_plain;h=4881d2ccdfc1bd298d1b46a078e2b023a3cc127e;p=dai-monitor.git

Initial distributed version
---

4881d2ccdfc1bd298d1b46a078e2b023a3cc127e
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..3fc2310
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+config.sh
+logs/
diff --git a/monitor-dai.sh b/monitor-dai.sh
new file mode 100755
index 0000000..5ef4954
--- /dev/null
+++ b/monitor-dai.sh
@@ -0,0 +1,212 @@
+#!/usr/bin/env bash
+
+# ================================================================
+#   Script to monitor DAI-live-streams (HLS streams)
+# ================================================================
+#
+#  Author: Kristian Kræmmer Nielsen <jkkn@tv2.dk>
+#
+#  Purpose:
+#     - Continuiously stream one HLS-stream
+#     - Look for CUE-IN and CUE-OUT markers
+#     - Log expected durations from CUE-OUT markers
+#     - Compare expected durations with actual duration until CUE-IN marker
+#     - Warn-log differencies over reasonable margin (TODO: reasonable margin)
+#     - Error-log if seeing duplicate CUE-OUT or CUE-IN markers
+#     - Optionally warn-log if seeing breaks over 12 minutes a hour (TODO)
+#
+# Errors goes to stderr
+# Warnings and info goes to stdout
+#
+# Syntax:
+#   monitor-dai.sh <url-to-hls-stream-including-tokens>
+#
+
+BASIC_CURL_PARAMS="--connect-timeout 10 --location --max-time 15 -Ss"
+SCRIPT_NAME=$(basename "$0")
+
+DATE_BSD="date -r "
+DATE_LINUX="date -d @"
+DATE="$DATE_BSD"
+
+# Used as default 10 seconds pause before retrying
+TARGET_DURATION=10
+
+# Time screw (expected delay in stream in seconds)
+STREAM_TIME_SCREW=105
+
+# Very limited function to resolve relative url
+function resolveRelativeURL() {
+    local relative="$1"
+    local full="$2"
+
+    # Actually FQ
+    re="^[a-z]+://.*"
+    if [[ "$relative" =~ $re ]]; then
+        echo "$relative"
+        return
+    fi
+    
+    # Absolute path
+    re="^/.*"
+    if [[ "$relative" =~ $re ]]; then
+        re="^([a-z]+://[^/]+).*"
+        if [[ "$full" =~ $re ]]; then
+            echo "${BASH_REMATCH[1]}${relative}"
+        else
+            echo >&2 "Unable to split url after hostname: $full"
+        fi
+        return
+    fi
+
+    # Relative - we will append and let curl fix later
+    echo "${full%/*}/${relative}"
+}
+
+function fatal() {
+    echo >&2 "Fatal error: $1"
+    exit 1
+}
+
+# Extract first stream from HLS Master-Playlist
+function resolveFirstStream() {
+   local hls="$1"
+   TMPFILE=$(mktemp -t "$SCRIPT_NAME") || fatal "Can not write tmp-file"
+   url=$(curl $BASIC_CURL_PARAMS -o "$TMPFILE" "$hls" -w "%{url_effective}")
+   if [ $? -eq 0 ]; then
+      while IFS="" read line; do
+        # First none comment is first element in playlist => we will use that stream
+        re="^[^#].*$"
+        if [[ "$line" =~ $re ]]; then
+            echo $(resolveRelativeURL "$line" "$url")
+            break
+        fi
+      done <"$TMPFILE"
+   else
+      echo >&2 "Unable to request: $hls" 
+   fi
+   rm "$TMPFILE"
+}
+
+# Monitor stream
+function monitorStream() {
+    local stream="$1"
+    local nextSegmentLength=0
+    local nextSeq=-1
+    local curSeq=-1
+    local expectedAdLength=-1
+    local mode="UNKNOWN"
+    local curLength=0
+
+    real_time=$(date +%s)
+    stream_time=$(($real_time - $STREAM_TIME_SCREW))
+    pretty_time=$($DATE ${stream_time%.*} +"%Y-%m-%d %H:%M:%S")
+
+    echo "NOTICE: Using time-offset of -$STREAM_TIME_SCREW seconds so time is $pretty_time, real time is $(date ${realtime} +"%Y-%m-%d %H:%M:%S")"
+    echo "$pretty_time: Started monitoring using: $stream"
+
+    while true; do
+        starttime=$(date +%s)
+        warn_not_a_playlist=1
+
+        TMPFILE=$(mktemp -t "$SCRIPT_NAME") || fatal "Can not write tmp-file"
+        curl $BASIC_CURL_PARAMS -o "$TMPFILE" "$stream"
+        if [ $? -eq 0 ]; then
+           while IFS="" read line; do
+
+             if [ "$line" == "#EXTM3U" ]; then
+                # Correct playlist
+                warn_not_a_playlist=0
+             fi
+
+             # Look for segment
+             re="^[^#].*"
+             if [[ "$line" =~ $re ]]; then
+                # Got segment
+                if [[ $curSeq -ne -1 ]]; then # we know media sequence
+                    if [[ $curSeq -ge $nextSeq ]]; then # later sequence than we need
+                        curLength=$(echo "$nextSegmentLength + $curLength" | bc)
+                        stream_time=$(echo "$stream_time + $nextSegmentLength" | bc)
+                        pretty_time=$($DATE ${stream_time%.*} +"%Y-%m-%d %H:%M:%S")
+                        if [[ $curSeq -ne $nextSeq ]]; then
+                            echo >&2 "$pretty_time: Skipped a segment by mistake ($curSeq > $nextSeq)"
+                        fi
+                        #echo >&2 "$pretty_time: Current mode length: $curLength"
+                        nextSeq=$(($curSeq + 1))
+                    fi
+                    curSeq=$(($curSeq + 1))
+                fi
+             fi
+
+             # Look for tags
+             re="^#([^:]+):?([0-9.]*)"
+             if [[ "$line" =~ $re ]]; then
+                tag="${BASH_REMATCH[1]}"
+                value="${BASH_REMATCH[2]}"
+                if [ "$tag" == "EXT-X-TARGETDURATION" ]; then
+                    if [ "$value" != "$TARGET_DURATION" ]; then
+                        echo "$pretty_time: Changing target durationt to: $value"
+                        TARGET_DURATION="$value"
+                    fi
+                elif [ "$tag" == "EXTINF" ]; then
+                    nextSegmentLength=$value
+                elif [ "$tag" == "EXT-X-MEDIA-SEQUENCE" ]; then
+                    #echo "$pretty_time: Got sequence no: $value"
+                    curSeq=$value
+                    if [ $nextSeq -le 0 ]; then
+                        nextSeq=$value
+                    fi
+                fi
+                if [[ $curSeq -ge $nextSeq ]]; then # we are interested in this tag
+                    if [ "$tag" == "EXT-X-CUE-OUT" ]; then
+                        # Begin ad block
+                        echo "$pretty_time: Seeing CUE-OUT (Ad block start) of duration: $value"
+                        if [ "$mode" == "ADBLOCK" ]; then
+                            echo >&2 "$pretty_time: ERROR: Already in ad block - extra cue-out after $curLength"
+                        else
+                            echo "$pretty_time: Ad-block started after $curLength seconds"
+                        fi
+                        mode="ADBLOCK"
+                        expectedAdLength=$value
+                        curLength=0
+                    elif [ "$tag" == "EXT-X-CUE-IN" ]; then
+                        # End ad block
+                        echo "$pretty_time: Seeing CUE-IN (Ad block end) after duration: $curLength"
+                        if [ "$mode" == "ADBLOCK" ]; then
+                            if [ "$expectedAdLength" != "$curLength" ]; then
+                                echo >&2 "$pretty_time: WARN: Block was not the length expected ($curLength <> $expectedAdLength)"
+                            fi
+                        elif [ "$mode" == "LIVE" ]; then
+                            echo >&2 "$pretty_time: ERROR: Extra CUE-IN outside Ad-block after $curLength"
+                        fi
+                        mode="LIVE"
+                        curLength=0
+                    fi
+                fi
+             fi
+           done <"$TMPFILE"
+
+           if [ $warn_not_a_playlist -eq 1 ]; then
+                echo >&2 "$pretty_time: ERROR: Not a valid playlist ($stream)"
+           fi
+            
+        else
+           echo >&2 "$pretty_time: Unable to request: $stream" 
+        fi
+        rm "$TMPFILE"
+
+        endtime=$(date +%s)
+        waittime=$(($TARGET_DURATION - ($endtime-$starttime)))
+        if [ "$waittime" -gt 0 ]; then
+            sleep $waittime
+        fi
+    done
+}
+
+streamurl=$(resolveFirstStream "$1")
+if [ -e "$streamurl" ]; then
+    echo >&2 "Invalid HLS-stream: $1"
+fi
+
+
+monitorStream "$streamurl"
diff --git a/start-monitors.sh b/start-monitors.sh
new file mode 100755
index 0000000..b3d5a37
--- /dev/null
+++ b/start-monitors.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+# ================================================================
+#   Starts up monitor script per stream
+# ================================================================
+#
+
+#  Author: Kristian Kræmmer Nielsen <jkkn@tv2.dk>
+
+myjobs=()
+
+
+function stop_jobs() {
+    echo "Stopping jobs..."
+    kill "${myjobs[@]}" 
+    wait
+    echo "Done."
+}
+
+function fatal() {
+    echo >2 "$1"
+    exit 1
+}
+
+## MAIN
+
+cd $(dirname $0)
+source "config.sh" || fatal "Missing config.sh"
+
+trap "stop_jobs" EXIT
+
+if [ ! -d "$log_dir" ]; then
+    mkdir -p "$log_dir" || exit 1
+fi
+
+for stream in "${streams[@]}"; do
+
+    NAME="" URL=""
+    declare ${stream}
+    echo "Starting monitor for $NAME ..."
+    ./monitor-dai.sh "$URL" >>"logs/$NAME.log" 2>&1 &
+    myjobs+=($!)
+
+done
+
+echo "All started."
+
+wait
+
+echo "Stopped unexpected."