#!@zsh@/bin/zsh alwaysTranscode=false ffmpeg() { { { @ffmpeg@/bin/ffmpeg $@ 1>&3 } 2>&1 | stdbuf -o 0 tr '\r' '\n' | grep -v --line-buffered -E '^$' 1>&2 } 3>&1 } pv() { { { @pv@/bin/pv -D 2 -i 2 -w 100 -H 1 -f $@ 1>&3 } 2>&1 | stdbuf -o 0 tr '\r' '\n' | grep -v --line-buffered -E '^$' 1>&2 } 3>&1 } trimName() { prefix=$1; shift; printf "%s%s" ${prefix} $(awk -v len=$((19 - $#prefix)) '{ if (length($0) > len) print substr($0, 1, len-3) "..."; else print; }' <<<"${(j: :)@}")} function notmuch { while true; do result=$(env NOTMUCH_CONFIG=${HOME}/.notmuch-rss-config @notmuch@/bin/notmuch "$@" 2>&1) if ! [[ $result =~ "already locked" ]]; then echo -nE $result return fi sleep 2 done } logTag=${0:t} exec 1> >(logger -t "$logTag" -p news.notice) exec 2> >(logger -t "$logTag" -p news.error) debug() { logger -t "$logTag" -p news.debug } warn() { logger -t "$logTag" -p news.warn } typeset -a cleanupCmds cleanupCmds=() function doCleanup { for cmd (${cleanupCmds}); do # print -- ${cmd} | debug eval $cmd | debug done } function cleanup() { local cmd cmd="" for arg ($@); do [[ -n ${cmd} ]] && cmd="$cmd " cmd="${cmd}${(qq)arg}" done cleanupCmds+=(${cmd}) } mungefilename () { tr "\`\"' /[:upper:]" "_____[:lower:]" <<<${@} | sed -r 's/[^0-9a-z\!\#\$\%\&\(\)\*\+\,\-\.\:\;\<\=\>\?\@\[\]\~\^\_\{\}\|]//g; s/_+/_/g' } [[ $#@ -le 0 || $#@ -gt 1 ]] && exit 2 msgId=$1 message=$(notmuch search --output=files --duplicate=1 $msgId) [[ ! -e $message ]] && exit 1 printf ">>> %s <<<\n %s\n" "${msgId}" "${message}" ( print " waiting for lock..." if ! flock -xn 9; then print " could not acquire lock." exit 0 fi print " locked." trap doCleanup EXIT typeset -a msgTags msgTags=($(notmuch search --output=tags $msgId )) if [[ ${msgTags[(i)cached]} -le $#msgTags ]]; then print "Message in Cache" | warn exit 0 fi if [[ 'base64' == $(sed '/^Content-Transfer-Encoding: */!d; s///;q' ${message}) ]]; then tmpFile=$(mktemp --tmpdir=/home/gkleen/rss/tmp .writeOut.XXXXXX) printf "Decoding base64 message content for ‘%s’" ${message} | debug sed '/^Content-Transfer-Encoding: */d; /^$/q' ${message} >! $tmpFile sed '1,/^$/d' ${message} | base64 -d >> $tmpFile mv -v $tmpFile ${message} fi from=$(mungefilename $(awk '/^From/ { gsub("^\"", "", $2); print $2; exit; }' "${message}")) if grep -q "
Enclosure: Enclosure: ]+href=[^">]*"([^"]+)".*$/\1/') else url=$(awk '/^X-RSS-URL/ { print $2; exit; }' "${message}") fi sld=$(cut -d '/' -f 3 <<<${url} | rev | cut -d '.' -f 1-2 | rev) if [[ -e ${HOME}/.dl-backup ]] && grep -q ${sld} ${HOME}/.dl-backup; then printf "Was told to back up off %s\n" ${sld} | warn exit 0 else printf "Proceeding on %s\n" ${sld} | debug fi dir="/srv/media/youtube" if ! { [[ -d $dir ]] || mkdir -p $dir }; then exit 1 fi formatString='bestvideo[width<=2560][height<=1440][fps<=60][vcodec=vp9]+bestaudio[acodec=opus]/bestvideo[width<=2560][height<=1440][fps<=60]+bestaudio/best[width<=2560][height<=1440][fps<=60]/best' if [[ ${msgTags[(i)tv]} -le $#msgTags ]]; then formatString="bestvideo[width<=2560][height<=1440][fps<=60][vcodec=h264]+bestaudio[acodec=aac]/bestvideo[width<=2560][height<=1440][fps<=60][vcodec=mpeg]+bestaudio[acodec=aac]/mp4[width<=2560][height<=1440][fps<=60][vcodec=h264][acodec=aac]/mp4[width<=2560][height<=1440][fps<=60][vcodec=mpeg][acodec=aac]/${formatString}" fi typeset -a args args=(--no-playlist --add-metadata --newline --mark-watched -f ${formatString}) @youtubedl@/bin/youtube-dl ${args} --get-filename -o $'%(extractor)s\n%(id)s\n%(format)s\n%(title)s\n%(ext)s' -- ${url} | { oldIFS=${IFS} export IFS= read extractor read videoId read format read title read fileExtension export IFS=${oldIFS} } || { ret=$? printf "An error occured while determining video metadata (exitcode: %d)\n" ${ret} >&2 notmuch tag +failed -- ${msgId} exit $? } if [[ "${extractor}" == "generic" ]]; then title=$(grep -E '^Subject' "${message}" | head -n 1 | cut -d ' ' -f 2-) fi filename="${dir}/${title}-${extractor}.${videoId}.${from}.${fileExtension}" filename=$(tr $'\n\0\/' ' ' <<<${filename} | tr -d $'!') printf "%s\n%s via %s: %s\nExpecting ‘%s’\n" ${title} ${videoId} ${extractor} ${format} ${filename:t} if [[ -n ${extractor} && -n ${videoId} ]]; then printf "Searching for %s on %s in cache...\n" ${videoId} ${extractor} | debug typeset -a messages messages=($(notmuch search --output=messages -- from:${extractor} and ${videoId} and is:cached)) if [[ ${#messages} -ne 0 ]]; then printf "Video already cached: %s\n" "${messages[*]}" | warn else printf "Video not yet cached\n" | debug fi fi @youtubedl@/bin/youtube-dl $args -o "${filename:r}.%(ext)s" -- ${url} 2>&1 &1 | \ grep 'max_volume' | sed -r 's/^.*max_volume: ([-0-9\.]+) dB$/\1/' ) printf "Maximum volume: %.2fdB\n" "${max_vol}" [[ -n "${max_vol}" ]] || max_vol=0 bare_amp=$(printf "%.2f" $(($max_vol * (-1)))) amp=$(printf "volume=%sdB" "${bare_amp}") typeset -a extensions extensions=("mkv" "mp3" "mp4" "webm") if [[ ${msgTags[(i)tv]} -le $#msgTags ]]; then extensions=("mp4") fi if [[ "${bare_amp}" -ne 0 || ${extensions[(i)${filename:e}]} -gt ${#extensions} ]]; then printf "Transcoding ‘%s’" ${title} | debug printf "%d %d %d/%d(%d)…\n" \ $([[ $($alwaysTranscode; print $?) -eq 0 ]]; print $?) \ $([[ "${bare_amp}" -ne 0 ]]; print $?) \ ${extensions[(i)${filename:e}]} ${#extensions} \ $([[ ${extensions[(i)${filename:e}]} -gt ${#extensions} ]]; print $?) \ | debug tempfile=$(mktemp --tmpdir=${filename:h} .transcode.${filename:t:r}.$$.XXXXXX.${filename:e}) cleanup rm -v -- "${tempfile}" mv -vf "${filename}" "${tempfile}" | debug typeset -A fileInfo for line ($(ffprobe -v error -show_format -show_streams -show_entries stream=codec_name,codec_type:format=:stream_tags=:stream_disposition=:format_tags= -of flat=h=0 -- ${tempfile})); do fileInfo[${line%=*}]=${(Q)line#*=} done ext=${extensions[1]} filename=${filename%.*}.${ext} typeset -a args args=(-y) typeset -a prePass prePass=() typeset -a cargs cargs=() typeset -a p1args p1args=() typeset -a p2args p2args=() for formatKey (${(k)fileInfo}); do [[ ${formatKey} =~ "stream.([0-9]+).codec_type" ]] || continue local i=$match[1] local cType=${fileInfo[${formatKey}]} local cName=${fileInfo[stream.${i}.codec_name]} printf "Stream %d: %s (%s)\n" ${i} ${cName} ${cType} | debug if [[ ${cType} == "video" ]]; then if [[ ${ext} == "mkv" ]]; then case ${cName} in h264|vp9|png) cargs+=(-c:${i} copy) ;; *) # p1args=(-pass 1 -threads 8 -speed 4 -tile-columns 6 -frame-parallel 1) # prePass+=(-c:${i} libvpx-vp9 -b:${i} 0 -crf:${i} 33) # p2args+=(-pass 2 -threads 8 -speed 2 -tile-columns 6 -frame-parallel 1 -auto-alt-ref 1 -lag-in-frames 25) # cargs+=(-c:${i} libvpx-vp9 -b:${i} 0 -crf:${i} 33) cargs+=(-c:${i} libvpx-vp9 -b:${i} 2M -threads 8 -tile-columns 6 -frame-parallel 1) ;; esac elif [[ ${ext} == "mp4" ]]; then case ${cName} in mpeg|h264) cargs+=(-c:${i} copy) ;; *) p2args+=(-strict -2) cargs+=(-c:${i} libx264) ;; esac fi elif [[ ${cType} == "audio" ]]; then if [[ ${ext} == "mkv" ]]; then if [[ ( ${cName} == opus || ${cName} == flac || ${cName} == vorbis ) && "${bare_amp}" -eq 0 ]]; then cargs+=(-c:${i} copy) else p2args+=(-vbr on -compression_level 10) cargs+=(-c:${i} libopus -b:${i} 256K) fi elif [[ ${ext} == "mp4" ]]; then if [[ ${cName} == aac && "${bare_amp}" -eq 0 ]]; then cargs+=(-c:${i} copy) else cargs+=(-c:${i} aac) fi fi fi done [[ "${bare_amp}" -ne 0 ]] && p2args+=(-af "${amp}") if [[ $#prePass -gt 0 ]]; then args+=(-v info -i ${tempfile}) oldPwd=${PWD} cd $(mktemp -d --tmpdir "transcode.${0:t}.$$.XXXXXX") cleanup rm -rfv -- ${PWD} p1args+=(-an -f matroska) print -- ${prePass} ${p1args} | debug ffmpeg ${args} ${prePass} ${p1args} -- /dev/null print -- ${cargs} ${p2args} | debug ffmpeg ${args} ${cargs} ${p2args} -- "${filename}" cd ${oldPwd} else args+=(-v warning -i pipe:0) print -- ${cargs} ${p2args} | debug pv -N "$(trimName "trans:" ${title})" ${tempfile} | ffmpeg ${args} ${cargs} ${p2args} -- "${filename}" fi fi chmod -v 644 "${filename}" | debug tmpFile=$(mktemp --tmpdir=/home/gkleen/rss/tmp .insertUrl.$$.XXXXXX) relUrl=$(realpath --relative-to=/srv/media ${filename}) typeset -a relUrlComponents relUrlComponents=(${(s./.)relUrl}) relUrl="" for urlPiece (${relUrlComponents}); do [[ -n "${relUrl}" ]] && relUrl+="/" relUrl+=$(sed -f ${HOME}/url_escape.sed <<<${urlPiece}) done awk -v "link=http://odin.asgard.yggdrasil/${relUrl}" '{ if (r == 0) { r = gsub("href=\"[^\"]+\"", "href=\"" link "\""); }; print; }' "${message}" >! $tmpFile mv -v $tmpFile $message | debug notmuch tag '+cached' -- $msgId && printf "Tagged ‘%s’ as 'cached'" ${msgId} if [[ -n "$(notmuch search "tag:inbox AND $msgId")" ]]; then if [[ ${msgTags[(i)tv]} -le $#msgTags ]]; then printf "%s\n%s\n" "Media available on odin" "${title}" \ | uux -p -n 'hel!notify-gkleen' -a download_youtube -u normal \ && print "Sent notification to hel via uucp" else queue.hel "${filename}" && notmuch tag '-inbox' '-unread' -- $msgId fi else print "Message vanished from inbox" | warn fi else printf "An error occured while downloading video at ‘%s’ (exitcode: %d)\n" ${url} ${ytResult} >&2 # notmuch tag +failed -- ${msgId} # exit $? fi ) 9<>"${message}"