From 359599d1ef5b6eb4bec1761eb541e3e2adbd3e72 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Kristian=20Kr=C3=A6mmer=20Nielsen?= Date: Fri, 31 Mar 2017 13:09:35 +0200 Subject: [PATCH] New features: - Token generation inline - Delays tuned per channel - Fail if skipping segments to align time - Write pid file --- .gitignore | 1 + monitor-dai.sh | 71 ++++++++++++++++++++++++++++++++++++----------- start-monitors.sh | 40 ++++++++++++++++++++++++-- 3 files changed, 93 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index 932811b..5d49648 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ config.sh logs/ playlists/ recordings/ +start-monitors.pid diff --git a/monitor-dai.sh b/monitor-dai.sh index 634bc4d..dee1481 100755 --- a/monitor-dai.sh +++ b/monitor-dai.sh @@ -15,14 +15,15 @@ # - Warn-log differencies over reasonable margin (TODO: reasonable margin) # - Error-log if seeing duplicate CUE-OUT or CUE-IN markers # - Optionally warn-log if seeing breaks over 12 minutes a hour (TODO) +# - Synchronize time by lowering target window to 1 second until first changed playlist (syncs to ~1 sec precision) # - Records first seen ad-block including one segment before and after -# - very useful for synchronization +# - very useful for synchronization of SCTE-35/SCTE-104 markers # -# Errors goes to stderr -# Warnings and info goes to stdout +# Log (info, warnings and errors) goes to stderr +# Playlist goes to stdout # # Syntax: -# monitor-dai.sh +# monitor-dai.sh # BASIC_CURL_PARAMS="--connect-timeout 10 --location --max-time 60 -Ss" @@ -40,9 +41,6 @@ fi # Used as default 10 seconds pause before retrying TARGET_DURATION=10 -# Time screw (expected delay in stream in seconds) -STREAM_TIME_SCREW=105 - # No adblocks to record RECORD_AD_BLOCKS=3 @@ -103,6 +101,7 @@ function resolveFirstStream() { function monitorStream() { local stream="$1" local recordFolder="$2" + local streamDelay="$3" local nextSegmentLength=0 local nextSeq=-1 local curSeq=-1 @@ -115,14 +114,21 @@ function monitorStream() { local tags="" local lastUsedSegment="" lastUsedStreamTime=0 # Stored for use of recording previous segment when seeing ad block local doRecord=0 doStopRecord=0 + local syncMode=-1 # (-1=get first, x=saw last at, 0=synched) In syncmode we try to request every second until we see a change to get a more precise time sync real_time=$(date +%s) - stream_time=$(($real_time - $STREAM_TIME_SCREW)) - pretty_time=$($DATE${stream_time%.*} +"%Y-%m-%d %H:%M:%S") - echo >&2 "NOTICE: Using time-offset of -$STREAM_TIME_SCREW seconds so time is $pretty_time, real time is $(date ${realtime} +"%Y-%m-%d %H:%M:%S")" - echo >&2 "$pretty_time: INFO: Started monitoring using: $stream" + if [ -n "$streamDelay" ]; then + stream_time=$(($real_time - $streamDelay)) + pretty_time=$($DATE${stream_time%.*} +"%Y-%m-%d %H:%M:%S") + echo >&2 "NOTICE: Using time-offset of -$streamDelay seconds so time is $pretty_time, real time is $(date ${realtime} +"%Y-%m-%d %H:%M:%S")" + else + stream_time="$real_time" + pretty_time=$($DATE${stream_time%.*} +"%Y-%m-%d %H:%M:%S") + fi + echo >&2 "$pretty_time: INFO: Started monitoring using: $stream" + while true; do starttime=$(date +%s) warn_not_a_playlist=1 @@ -160,7 +166,8 @@ function monitorStream() { stream_time=$(echo "$stream_time + $nextSegmentLength" | bc) pretty_time=$($DATE${stream_time%.*} +"%Y-%m-%d %H:%M:%S") if [[ $curSeq -ne $nextSeq ]]; then - echo >&2 "$pretty_time: WARN: Lost segments ($curSeq > $nextSeq)" + echo >&2 "$pretty_time: WARN: Lost segments ($curSeq > $nextSeq)... resetting..." + failed=10 fi #echo >&2 "$pretty_time: Current mode length: $curLength" nextSeq=$(($curSeq + 1)) @@ -168,9 +175,25 @@ function monitorStream() { # Log stopped recording (after pretty_time moved forward) if [ "$doStopRecord" == "1" ]; then doRecord=0 - echo >&2 "$pretty_time: INFO: Stopped recording." + #echo >&2 "$pretty_time: INFO: Stopped recording." doStopRecord=0 fi + # Sync time a bit better by using target duration of 1 second in beginning + if [ "$syncMode" -gt 0 ]; then + real_time="$starttime" + syncOffset="$(echo "$target_duration - ($real_time - $syncMode)" | bc)" + if [ "$syncOffset" -gt 0 ]; then + stream_time="$(echo "$stream_time - $syncOffset" | bc)" + pretty_time=$($DATE${stream_time%.*} +"%Y-%m-%d %H:%M:%S") + syncMode=0 + echo >&2 "$pretty_time: INFO: Time was adjusted by -$syncOffset seconds." + elif [ "$syncOffset" -eq 0 ]; then + syncMode=0 + else + # Resync + syncMode=-1 + fi + fi fi curSeq=$(($curSeq + 1)) fi @@ -246,6 +269,11 @@ function monitorStream() { failed=$(($failed + 1)) else failed=0 + + # Use this time for sync offset? + if [ $syncMode -eq -1 ]; then + syncMode="$starttime" + fi fi else @@ -255,13 +283,24 @@ function monitorStream() { rm "$TMPFILE" endtime=$(date +%s) - waittime=$(($target_duration - ($endtime-$starttime))) + if [ "$syncMode" -ne 0 ]; then + # While syncing we only wait one second + waittime=$((1 - ($endtime-$starttime))) + else + waittime=$(($target_duration - ($endtime-$starttime))) + fi if [ "$waittime" -gt 0 ]; then sleep $waittime fi + if [[ "$failed" -gt 0 && "$syncMode" -ne 0 ]]; then + syncMode=-1 + fi + if [ "$failed" -ge 2 ]; then - echo >&2 "$pretty_time: ERROR: Stream failed twice.. resetting..." + if [ "$failed" -ne 10 ]; then + echo >&2 "$pretty_time: ERROR: Stream failed twice.. resetting..." + fi return fi done @@ -317,7 +356,7 @@ while true; do echo >&2 "INFO: (will retry in 10 seconds)" sleep 10 else - monitorStream "$streamurl" "$2" + monitorStream "$streamurl" "$2" "$3" fi done diff --git a/start-monitors.sh b/start-monitors.sh index 4d23069..a05ff3c 100755 --- a/start-monitors.sh +++ b/start-monitors.sh @@ -21,6 +21,7 @@ function stop_jobs() { kill "${myjobs[@]}" wait echo "Done." + rm "$MYPID" } function fatal() { @@ -28,6 +29,38 @@ function fatal() { exit 1 } +# Token generators +function addLevel3Tokens() { + local secret="$1" + local secretno="$2" + shift 2 + while [ -n "$1" ]; do + if [[ "$1" =~ ^((.+[[:space:]])URL=([^[:space:]]+))([[:space:]].+)?$ ]]; then + echo -n "${BASH_REMATCH[1]}?token=${secretno}" + echo -n "${BASH_REMATCH[3]}" | sed -Ee 's/.*https?:\/\/[^/]*//' | openssl sha1 -hmac "$secret" | sed -e 's/(stdin)= //' | cut -c1-20 + echo "${BASH_REMATCH[4]}" + else + echo "$1" + fi + shift + done +} + + +function addDaiTokens() { + local secret="$1" + shift + while [ -n "$1" ]; do + if [[ "$1" =~ ^((.+[[:space:]])URL=([^[:space:]]+))([[:space:]].+)?$ ]]; then + echo "${BASH_REMATCH[1]}?api-key=$secret${BASH_REMATCH[4]}" + else + echo "$1" + fi + shift + done +} + + ## MAIN cd $(dirname $0) @@ -52,16 +85,17 @@ fi for stream in "${streams[@]}"; do - NAME="" URL="" + NAME="" URL="" DELAY="" declare ${stream} - echo "Starting monitor for $NAME ..." - ./monitor-dai.sh "$URL" >"$playlist_dir/$NAME.m3u8" 2>"$log_dir/$NAME.log" "$recordings_dir/$NAME/" & + echo "Starting monitor for $NAME ($URL) ..." + ./monitor-dai.sh "$URL" "$recordings_dir/$NAME/" "$DELAY" >"$playlist_dir/$NAME.m3u8" 2>"$log_dir/$NAME.log" & myjobs+=($!) done echo "All started." echo "(To shutdown all run: kill $BASHPID)" + echo "$BASHPID" >"$MYPID" exec 1>&- -- 2.52.0