From 04fe40544c0ef912bdd72a4146f88827eb3bf5f9 Mon Sep 17 00:00:00 2001 From: Gregor Kleen Date: Fri, 6 Apr 2018 14:17:41 +0200 Subject: strm --- odin/strm/cleanup_youtube | 61 ++++++++ odin/strm/download_youtube | 337 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 398 insertions(+) create mode 100755 odin/strm/cleanup_youtube create mode 100755 odin/strm/download_youtube (limited to 'odin/strm') diff --git a/odin/strm/cleanup_youtube b/odin/strm/cleanup_youtube new file mode 100755 index 00000000..0df15d0b --- /dev/null +++ b/odin/strm/cleanup_youtube @@ -0,0 +1,61 @@ +#!@zsh@/bin/zsh + +function notmuch { + while true; do + result=$($_env NOTMUCH_CONFIG=${HOME}/.notmuch-rss-config @notmuch@/bin/notmuch "$@" 2>&1) + if ! [[ $result =~ "already locked" ]]; then + echo -nE $result + return + fi + sleep 2 + done +} + +tagFile="" + +function cleanup { + [[ -n "${tagFile}" ]] && rm -fv ${tagFile} +} + +dir=/srv/media/youtube +maxsize=$((1024 * 1024 * 1024 * 50)) + +while [[ $(du -bs $dir | awk '{ print $1; }') -gt $maxsize ]]; do + find $dir -type f -not -path $dir/CACHEDIR.TAG -print0 | \ + xargs -0 -- $_stat --printf '%W %Y %Z %X %n\0' | \ + sort -t '\0' -z | \ + awk -F '\0' '{ gsub("^\\S+\\s+\\S+\\s+\\S+\\s+\\S+\\s+", "", $1); printf "%s\0", $1; }' | \ + xargs -0 -- rm -v +done + +find $dir -xtype l -delete +find $dir -type d -empty -delete + +notmuch search --output=messages --format=text 'is:cached' | \ + while read id; do + untag=false + url=$(notmuch show --format=raw "$id" | grep 'http://odin.asgard.yggdrasil/youtube' | sed -r 's|^.*href="http://odin.asgard.yggdrasil/youtube/([^"]+)".*$|\1|' | sed -f /usr/lib/url_unescape.sed) + + printf "%s\n ‘%s’\n" ${id} ${dir}/${url} + + if [[ -z "${url}" ]]; then + printf " Could not extract filename.\n" + untag=true + fi + + if [[ -n "${url}" && ! -e "${dir}/${url}" ]]; then + printf " File vanished\n" + untag=true + fi + + if ${untag}; then + if [[ -z "${tagFile}" ]]; then + tagFile=$(mktemp --tmpdir $$.tags.XXXXXXXXXX) + printf "Using %s\n" ${tagFile} + fi + + { printf "-cached -- %s\n" ${id} >> ${tagFile} } && printf " Tagging ‘-cached’...\n" + fi + done + +[[ -n "${tagFile}" ]] && notmuch tag --batch --input=${tagFile} diff --git a/odin/strm/download_youtube b/odin/strm/download_youtube new file mode 100755 index 00000000..33e9a454 --- /dev/null +++ b/odin/strm/download_youtube @@ -0,0 +1,337 @@ +#!@zsh@/bin/zsh + +alwaysTranscode=false + +ffmpeg() { { { @ffmpeg-full@/bin/ffmpeg $@ 1>&3 } 2>&1 | stdbuf -o 0 tr '\r' '\n' | grep -v --line-buffered -E '^$' 1>&2 } 3>&1 } +pv() { { { @pv@/bin/pv -D 2 -i 2 -w 100 -H 1 -f $@ 1>&3 } 2>&1 | stdbuf -o 0 tr '\r' '\n' | grep -v --line-buffered -E '^$' 1>&2 } 3>&1 } +trimName() { prefix=$1; shift; printf "%s%s" ${prefix} $(awk -v len=$((19 - $#prefix)) '{ if (length($0) > len) print substr($0, 1, len-3) "..."; else print; }' <<<"${(j: :)@}")} + +function notmuch { + while true; do + result=$(env NOTMUCH_CONFIG=${HOME}/.notmuch-rss-config @notmuch@/bin/notmuch "$@" 2>&1) + if ! [[ $result =~ "already locked" ]]; then + echo -nE $result + return + fi + sleep 2 + done +} + +logTag=${0:t} + +exec 1> >(logger -t "$logTag" -p news.notice) +exec 2> >(logger -t "$logTag" -p news.error) + +debug() { logger -t "$logTag" -p news.debug } +warn() { logger -t "$logTag" -p news.warn } + +typeset -a cleanupCmds +cleanupCmds=() + +function doCleanup { + for cmd (${cleanupCmds}); do + # print -- ${cmd} | debug + eval $cmd | debug + done +} + +function cleanup() { + local cmd + cmd="" + for arg ($@); do + [[ -n ${cmd} ]] && cmd="$cmd " + cmd="${cmd}${(qq)arg}" + done + + cleanupCmds+=(${cmd}) +} + +mungefilename () +{ + tr "\`\"' /[:upper:]" "_____[:lower:]" <<<${@} | sed -r 's/[^0-9a-z\!\#\$\%\&\(\)\*\+\,\-\.\:\;\<\=\>\?\@\[\]\~\^\_\{\}\|]//g; s/_+/_/g' +} + +[[ $#@ -le 0 || $#@ -gt 1 ]] && exit 2 + +msgId=$1 +message=$(notmuch search --output=files --duplicate=1 $msgId) +[[ ! -e $message ]] && exit 1 + +printf ">>> %s <<<\n %s\n" "${msgId}" "${message}" + +( print " waiting for lock..." + + if ! flock -xn 9; then + print " could not acquire lock." + exit 0 + fi + + print " locked." + + trap doCleanup EXIT + + typeset -a msgTags + msgTags=($(notmuch search --output=tags $msgId )) + + if [[ ${msgTags[(i)cached]} -le $#msgTags ]]; then + print "Message in Cache" | warn + exit 0 + fi + + if [[ 'base64' == $(sed '/^Content-Transfer-Encoding: */!d; s///;q' ${message}) ]]; then + tmpFile=$(mktemp --tmpdir=/home/gkleen/rss/tmp .writeOut.XXXXXX) + printf "Decoding base64 message content for ‘%s’" ${message} | debug + sed '/^Content-Transfer-Encoding: */d; /^$/q' ${message} >! $tmpFile + sed '1,/^$/d' ${message} | base64 -d >> $tmpFile + mv -v $tmpFile ${message} + fi + + from=$(mungefilename $(awk '/^From/ { gsub("^\"", "", $2); print $2; exit; }' "${message}")) + if grep -q "

Enclosure: Enclosure: ]+href=[^">]*"([^"]+)".*$/\1/') + else + url=$(awk '/^X-RSS-URL/ { print $2; exit; }' "${message}") + fi + + sld=$(cut -d '/' -f 3 <<<${url} | rev | cut -d '.' -f 1-2 | rev) + + if [[ -e ${HOME}/.dl-backup ]] && grep -q ${sld} ${HOME}/.dl-backup; then + printf "Was told to back up off %s\n" ${sld} | warn + exit 0 + else + printf "Proceeding on %s\n" ${sld} | debug + fi + + dir="/srv/media/youtube" + if ! { [[ -d $dir ]] || mkdir -p $dir }; then + exit 1 + fi + + formatString='bestvideo[width<=2560][height<=1440][fps<=60][vcodec=vp9]+bestaudio[acodec=opus]/bestvideo[width<=2560][height<=1440][fps<=60]+bestaudio/best[width<=2560][height<=1440][fps<=60]/best' + + if [[ ${msgTags[(i)tv]} -le $#msgTags ]]; then + formatString="bestvideo[width<=2560][height<=1440][fps<=60][vcodec=h264]+bestaudio[acodec=aac]/bestvideo[width<=2560][height<=1440][fps<=60][vcodec=mpeg]+bestaudio[acodec=aac]/mp4[width<=2560][height<=1440][fps<=60][vcodec=h264][acodec=aac]/mp4[width<=2560][height<=1440][fps<=60][vcodec=mpeg][acodec=aac]/${formatString}" + fi + + typeset -a args + args=(--no-playlist --add-metadata --newline --mark-watched -f ${formatString}) + + youtube-dl ${args} --get-filename -o $'%(extractor)s\n%(id)s\n%(format)s\n%(title)s\n%(ext)s' -- ${url} | { + oldIFS=${IFS} + export IFS= + + read extractor + read videoId + read format + read title + read fileExtension + + export IFS=${oldIFS} + } || { + ret=$? + printf "An error occured while determining video metadata (exitcode: %d)\n" ${ret} >&2 + notmuch tag +failed -- ${msgId} + exit $? + } + + if [[ "${extractor}" == "generic" ]]; then + title=$(grep -E '^Subject' "${message}" | head -n 1 | cut -d ' ' -f 2-) + fi + + filename="${dir}/${title}-${extractor}.${videoId}.${from}.${fileExtension}" + filename=$(tr $'\n\0\/' ' ' <<<${filename} | tr -d $'!') + + printf "%s\n%s via %s: %s\nExpecting ‘%s’\n" ${title} ${videoId} ${extractor} ${format} ${filename:t} + + if [[ -n ${extractor} && -n ${videoId} ]]; then + printf "Searching for %s on %s in cache...\n" ${videoId} ${extractor} | debug + + typeset -a messages + messages=($(notmuch search --output=messages -- from:${extractor} and ${videoId} and is:cached)) + + if [[ ${#messages} -ne 0 ]]; then + printf "Video already cached: %s\n" "${messages[*]}" | warn + else + printf "Video not yet cached\n" | debug + fi + fi + + youtube-dl $args -o "${filename:r}.%(ext)s" -- ${url} 2>&1 &1 | \ + grep 'max_volume' | sed -r 's/^.*max_volume: ([-0-9\.]+) dB$/\1/' + ) + printf "Maximum volume: %.2fdB\n" "${max_vol}" + [[ -n "${max_vol}" ]] || max_vol=0 + bare_amp=$(printf "%.2f" $(($max_vol * (-1)))) + amp=$(printf "volume=%sdB" "${bare_amp}") + + typeset -a extensions + extensions=("mkv" "mp3" "mp4" "webm") + + if [[ ${msgTags[(i)tv]} -le $#msgTags ]]; then + extensions=("mp4") + fi + + if [[ "${bare_amp}" -ne 0 || ${extensions[(i)${filename:e}]} -gt ${#extensions} ]]; then + printf "Transcoding ‘%s’" ${title} | debug + printf "%d %d %d/%d(%d)…\n" \ + $([[ $($alwaysTranscode; print $?) -eq 0 ]]; print $?) \ + $([[ "${bare_amp}" -ne 0 ]]; print $?) \ + ${extensions[(i)${filename:e}]} ${#extensions} \ + $([[ ${extensions[(i)${filename:e}]} -gt ${#extensions} ]]; print $?) \ + | debug + tempfile=$(mktemp --tmpdir=${filename:h} .transcode.${filename:t:r}.$$.XXXXXX.${filename:e}) + cleanup rm -v -- "${tempfile}" + mv -vf "${filename}" "${tempfile}" | debug + + typeset -A fileInfo + for line ($(ffprobe -v error -show_format -show_streams -show_entries stream=codec_name,codec_type:format=:stream_tags=:stream_disposition=:format_tags= -of flat=h=0 -- ${tempfile})); do + fileInfo[${line%=*}]=${(Q)line#*=} + done + + ext=${extensions[1]} + filename=${filename%.*}.${ext} + typeset -a args + args=(-y) + typeset -a prePass + prePass=() + typeset -a cargs + cargs=() + typeset -a p1args + p1args=() + typeset -a p2args + p2args=() + for formatKey (${(k)fileInfo}); do + [[ ${formatKey} =~ "stream.([0-9]+).codec_type" ]] || continue + local i=$match[1] + local cType=${fileInfo[${formatKey}]} + local cName=${fileInfo[stream.${i}.codec_name]} + + printf "Stream %d: %s (%s)\n" ${i} ${cName} ${cType} | debug + + if [[ ${cType} == "video" ]]; then + if [[ ${ext} == "mkv" ]]; then + case ${cName} in + h264|vp9|png) + cargs+=(-c:${i} copy) + ;; + *) + # p1args=(-pass 1 -threads 8 -speed 4 -tile-columns 6 -frame-parallel 1) + # prePass+=(-c:${i} libvpx-vp9 -b:${i} 0 -crf:${i} 33) + # p2args+=(-pass 2 -threads 8 -speed 2 -tile-columns 6 -frame-parallel 1 -auto-alt-ref 1 -lag-in-frames 25) + # cargs+=(-c:${i} libvpx-vp9 -b:${i} 0 -crf:${i} 33) + cargs+=(-c:${i} libvpx-vp9 -b:${i} 2M -threads 8 -tile-columns 6 -frame-parallel 1) + ;; + esac + elif [[ ${ext} == "mp4" ]]; then + case ${cName} in + mpeg|h264) + cargs+=(-c:${i} copy) + ;; + *) + p2args+=(-strict -2) + cargs+=(-c:${i} libx264) + ;; + esac + fi + elif [[ ${cType} == "audio" ]]; then + if [[ ${ext} == "mkv" ]]; then + if [[ ( ${cName} == opus || ${cName} == flac || ${cName} == vorbis ) && "${bare_amp}" -eq 0 ]]; then + cargs+=(-c:${i} copy) + else + p2args+=(-vbr on -compression_level 10) + cargs+=(-c:${i} libopus -b:${i} 256K) + fi + elif [[ ${ext} == "mp4" ]]; then + if [[ ${cName} == aac && "${bare_amp}" -eq 0 ]]; then + cargs+=(-c:${i} copy) + else + cargs+=(-c:${i} aac) + fi + fi + fi + done + + [[ "${bare_amp}" -ne 0 ]] && p2args+=(-af "${amp}") + + if [[ $#prePass -gt 0 ]]; then + args+=(-v info -i ${tempfile}) + + oldPwd=${PWD} + cd $(mktemp -d --tmpdir "transcode.${0:t}.$$.XXXXXX") + cleanup rm -rfv -- ${PWD} + + p1args+=(-an -f matroska) + print -- ${prePass} ${p1args} | debug + ffmpeg ${args} ${prePass} ${p1args} -- /dev/null + + print -- ${cargs} ${p2args} | debug + ffmpeg ${args} ${cargs} ${p2args} -- "${filename}" + + cd ${oldPwd} + else + args+=(-v warning -i pipe:0) + + print -- ${cargs} ${p2args} | debug + pv -N "$(trimName "trans:" ${title})" ${tempfile} | ffmpeg ${args} ${cargs} ${p2args} -- "${filename}" + fi + fi + + chmod -v 644 "${filename}" | debug + tmpFile=$(mktemp --tmpdir=/home/gkleen/rss/tmp .insertUrl.$$.XXXXXX) + relUrl=$(realpath --relative-to=/srv/media ${filename}) + typeset -a relUrlComponents + relUrlComponents=(${(s./.)relUrl}) + relUrl="" + for urlPiece (${relUrlComponents}); do + [[ -n "${relUrl}" ]] && relUrl+="/" + relUrl+=$(sed -f ${HOME}/url_escape.sed <<<${urlPiece}) + done + awk -v "link=http://odin.asgard.yggdrasil/${relUrl}" '{ if (r == 0) { r = gsub("href=\"[^\"]+\"", "href=\"" link "\""); }; print; }' "${message}" >! $tmpFile + mv -v $tmpFile $message | debug + notmuch tag '+cached' -- $msgId && printf "Tagged ‘%s’ as 'cached'" ${msgId} + + if [[ -n "$(notmuch search "tag:inbox AND $msgId")" ]]; then + if [[ ${msgTags[(i)tv]} -le $#msgTags ]]; then + printf "%s\n%s\n" "Media available on odin" "${title}" \ + | uux -p -n 'hel!notify-gkleen' -a download_youtube -u normal \ + && print "Sent notification to hel via uucp" + else + queue.hel "${filename}" && notmuch tag '-inbox' '-unread' -- $msgId + fi + else + print "Message vanished from inbox" | warn + fi + else + printf "An error occured while downloading video at ‘%s’ (exitcode: %d)\n" ${url} ${ytResult} >&2 + # notmuch tag +failed -- ${msgId} + # exit $? + fi +) 9<>"${message}" -- cgit v1.2.3