strm

4 files changed, 422 insertions, 0 deletions

diff --git a/odin/strm.nix b/odin/strm.nix
new file mode 100644
index 00000000..33e605af
--- /dev/null
+++ b/odin/strm.nix
@@ -0,0 +1,21 @@
+{ stdenv, zsh, ffmpeg-full, youtube-dl, pv, notmuch }:
+
+stdenv.mkDerivation {
+  name = "strm";
+  src = ./strm;
+
+  phases = [ "unpackPhase" "buildPhase" "installPhase" ];
+
+  inherit zsh ffmpeg-full youtube-dl pv notmuch;
+
+  buildPhase = ''
+    substituteAllInPlace download_youtube
+    substituteAllInPlace cleanup_youtube
+  '';
+
+  installPhase = ''
+    mkdir -p $out/bin
+
+    install -m 755 -t $out/bin download_youtube cleanup_youtube
+  '';
+}
diff --git a/odin/strm/cleanup_youtube b/odin/strm/cleanup_youtube
new file mode 100755
index 00000000..0df15d0b
--- /dev/null
+++ b/odin/strm/cleanup_youtube
@@ -0,0 +1,61 @@
+#!@zsh@/bin/zsh
+
+function notmuch {
+    while true; do 
+        result=$($_env NOTMUCH_CONFIG=${HOME}/.notmuch-rss-config @notmuch@/bin/notmuch "$@" 2>&1)
+        if ! [[ $result =~ "already locked" ]]; then
+            echo -nE $result
+            return
+        fi
+        sleep 2
+    done
+}
+
+tagFile=""
+
+function cleanup {
+    [[ -n "${tagFile}" ]] && rm -fv ${tagFile}
+}
+
+dir=/srv/media/youtube
+maxsize=$((1024 * 1024 * 1024 * 50))
+
+while [[ $(du -bs $dir | awk '{ print $1; }') -gt $maxsize ]]; do
+    find $dir -type f -not -path $dir/CACHEDIR.TAG -print0 | \
+        xargs -0 -- $_stat --printf '%W %Y %Z %X %n\0' | \
+        sort -t '\0' -z | \
+        awk -F '\0' '{ gsub("^\\S+\\s+\\S+\\s+\\S+\\s+\\S+\\s+", "", $1); printf "%s\0", $1; }' | \
+        xargs -0 -- rm -v
+done
+
+find $dir -xtype l -delete
+find $dir -type d -empty -delete
+
+notmuch search --output=messages --format=text 'is:cached' | \
+    while read id; do 
+        untag=false
+        url=$(notmuch show --format=raw "$id" | grep 'http://odin.asgard.yggdrasil/youtube' | sed -r 's|^.*href="http://odin.asgard.yggdrasil/youtube/([^"]+)".*$|\1|' | sed -f /usr/lib/url_unescape.sed)
+
+        printf "%s\n  ‘%s’\n" ${id} ${dir}/${url}
+        
+        if [[ -z "${url}" ]]; then
+            printf "  Could not extract filename.\n"
+            untag=true
+        fi
+
+        if [[ -n "${url}" && ! -e "${dir}/${url}" ]]; then
+            printf "  File vanished\n"
+            untag=true
+        fi
+
+        if ${untag}; then
+            if [[ -z "${tagFile}" ]]; then
+                tagFile=$(mktemp --tmpdir $$.tags.XXXXXXXXXX)
+                printf "Using %s\n" ${tagFile}
+            fi
+            
+            { printf "-cached -- %s\n" ${id} >> ${tagFile} } && printf "  Tagging ‘-cached’...\n"
+        fi
+    done
+
+[[ -n "${tagFile}" ]] && notmuch tag --batch --input=${tagFile}
diff --git a/odin/strm/download_youtube b/odin/strm/download_youtube
new file mode 100755
index 00000000..33e9a454
--- /dev/null
+++ b/odin/strm/download_youtube
@@ -0,0 +1,337 @@
+#!@zsh@/bin/zsh
+
+alwaysTranscode=false
+
+ffmpeg() { { { @ffmpeg-full@/bin/ffmpeg $@ 1>&3 } 2>&1 | stdbuf -o 0 tr '\r' '\n' | grep -v --line-buffered -E '^$' 1>&2 } 3>&1 }
+pv() { { { @pv@/bin/pv -D 2 -i 2 -w 100 -H 1 -f $@ 1>&3 } 2>&1 | stdbuf -o 0 tr '\r' '\n' | grep -v --line-buffered -E '^$' 1>&2 } 3>&1 }
+trimName() { prefix=$1; shift; printf "%s%s" ${prefix} $(awk -v len=$((19 - $#prefix)) '{ if (length($0) > len) print substr($0, 1, len-3) "..."; else print; }' <<<"${(j: :)@}")}
+
+function notmuch {
+    while true; do 
+        result=$(env NOTMUCH_CONFIG=${HOME}/.notmuch-rss-config @notmuch@/bin/notmuch "$@" 2>&1)
+        if ! [[ $result =~ "already locked" ]]; then
+            echo -nE $result
+            return
+        fi
+        sleep 2
+    done
+}
+
+logTag=${0:t}
+
+exec 1> >(logger -t "$logTag" -p news.notice)
+exec 2> >(logger -t "$logTag" -p news.error)
+
+debug() { logger -t "$logTag" -p news.debug }
+warn() { logger -t "$logTag" -p news.warn }
+
+typeset -a cleanupCmds
+cleanupCmds=()
+
+function doCleanup {
+    for cmd (${cleanupCmds}); do
+        # print -- ${cmd} | debug
+        eval $cmd | debug
+    done
+}
+
+function cleanup() {
+    local cmd
+    cmd=""
+    for arg ($@); do
+        [[ -n ${cmd} ]] && cmd="$cmd "
+        cmd="${cmd}${(qq)arg}"
+    done
+
+    cleanupCmds+=(${cmd})
+}
+
+mungefilename ()
+{
+    tr "\`\"' /[:upper:]" "_____[:lower:]" <<<${@} | sed -r 's/[^0-9a-z\!\#\$\%\&\(\)\*\+\,\-\.\:\;\<\=\>\?\@\[\]\~\^\_\{\}\|]//g; s/_+/_/g'
+}
+
+[[ $#@ -le 0 || $#@ -gt 1 ]] && exit 2
+
+msgId=$1
+message=$(notmuch search --output=files --duplicate=1 $msgId)
+[[ ! -e $message ]] && exit 1
+
+printf ">>> %s <<<\n  %s\n" "${msgId}" "${message}"
+
+( print "  waiting for lock..."
+  
+  if ! flock -xn 9; then
+      print "  could not acquire lock."
+      exit 0
+  fi
+
+  print "  locked."
+
+  trap doCleanup EXIT
+
+  typeset -a msgTags
+  msgTags=($(notmuch search --output=tags $msgId ))
+
+  if [[ ${msgTags[(i)cached]} -le $#msgTags ]]; then
+      print "Message in Cache" | warn
+      exit 0
+  fi
+
+  if [[ 'base64' == $(sed '/^Content-Transfer-Encoding: */!d; s///;q' ${message}) ]]; then
+      tmpFile=$(mktemp --tmpdir=/home/gkleen/rss/tmp .writeOut.XXXXXX)
+      printf "Decoding base64 message content for ‘%s’" ${message} | debug
+      sed '/^Content-Transfer-Encoding: */d; /^$/q' ${message} >! $tmpFile
+      sed '1,/^$/d' ${message} | base64 -d >> $tmpFile
+      mv -v $tmpFile ${message}
+  fi
+
+  from=$(mungefilename $(awk '/^From/ { gsub("^\"", "", $2); print $2; exit; }' "${message}"))
+  if grep -q "<p>Enclosure: <a" "${message}"; then
+      url=$(tr -d '\n' < "${message}" | sed -r 's/^.*<p>Enclosure: <a[^>]+href=[^">]*"([^"]+)".*$/\1/')
+  else
+      url=$(awk '/^X-RSS-URL/ { print $2; exit; }' "${message}")
+  fi
+
+  sld=$(cut -d '/' -f 3 <<<${url} | rev | cut -d '.' -f 1-2 | rev)
+
+  if [[ -e ${HOME}/.dl-backup ]] && grep -q ${sld} ${HOME}/.dl-backup; then
+      printf "Was told to back up off %s\n" ${sld} | warn
+      exit 0
+  else
+      printf "Proceeding on %s\n" ${sld} | debug
+  fi
+
+  dir="/srv/media/youtube"
+  if ! { [[ -d $dir ]] || mkdir -p $dir }; then
+      exit 1
+  fi
+
+  formatString='bestvideo[width<=2560][height<=1440][fps<=60][vcodec=vp9]+bestaudio[acodec=opus]/bestvideo[width<=2560][height<=1440][fps<=60]+bestaudio/best[width<=2560][height<=1440][fps<=60]/best'
+
+  if [[ ${msgTags[(i)tv]} -le $#msgTags ]]; then
+      formatString="bestvideo[width<=2560][height<=1440][fps<=60][vcodec=h264]+bestaudio[acodec=aac]/bestvideo[width<=2560][height<=1440][fps<=60][vcodec=mpeg]+bestaudio[acodec=aac]/mp4[width<=2560][height<=1440][fps<=60][vcodec=h264][acodec=aac]/mp4[width<=2560][height<=1440][fps<=60][vcodec=mpeg][acodec=aac]/${formatString}"
+  fi
+
+  typeset -a args
+  args=(--no-playlist --add-metadata --newline --mark-watched -f ${formatString})
+
+  youtube-dl ${args} --get-filename -o $'%(extractor)s\n%(id)s\n%(format)s\n%(title)s\n%(ext)s' -- ${url} | {
+      oldIFS=${IFS}
+      export IFS=
+
+      read extractor
+      read videoId
+      read format
+      read title
+      read fileExtension
+
+      export IFS=${oldIFS}
+  } || {
+      ret=$?
+      printf "An error occured while determining video metadata (exitcode: %d)\n" ${ret} >&2
+      notmuch tag +failed -- ${msgId}
+      exit $?
+  }
+
+  if [[ "${extractor}" == "generic" ]]; then
+      title=$(grep -E '^Subject' "${message}" | head -n 1 | cut -d ' ' -f 2-)
+  fi
+
+  filename="${dir}/${title}-${extractor}.${videoId}.${from}.${fileExtension}"
+  filename=$(tr $'\n\0\/' ' ' <<<${filename} | tr -d $'!') 
+
+  printf "%s\n%s via %s: %s\nExpecting ‘%s’\n" ${title} ${videoId} ${extractor} ${format} ${filename:t}
+
+  if [[ -n ${extractor} && -n ${videoId} ]]; then
+      printf "Searching for %s on %s in cache...\n" ${videoId} ${extractor} | debug
+
+      typeset -a messages
+      messages=($(notmuch search --output=messages -- from:${extractor} and ${videoId} and is:cached))
+
+      if [[ ${#messages} -ne 0 ]]; then
+          printf "Video already cached: %s\n" "${messages[*]}" | warn
+      else
+          printf "Video not yet cached\n" | debug
+      fi
+  fi
+
+  youtube-dl $args -o "${filename:r}.%(ext)s" -- ${url} 2>&1 </dev/null | stdbuf -o0 tr '\r' '\n'
+  ytResult=$?
+
+  if [[ ! -e ${filename} && -n ${filename} ]]; then
+      for f (${filename:r}.*(N)); do
+          shouldDelete=false
+
+          grep -qE '\.f[0-9]+\.' <<<${f} && shouldDelete=true
+          grep -qE '\.part(-Frag[0-9]+)?$' <<<${f} && shouldDelete=true
+          grep -qE '\.temp\.[^\.]+$' <<<${f} && shouldDelete=true
+
+          if ${shouldDelete}; then
+              rm -v ${f}
+          else
+              filename=${f}
+          fi 
+      done
+  fi
+
+  [[ -n ${filename} ]] && printf "Found ‘%s’\n" ${filename:t} || debug
+
+  # newFilename=${filename:h}/$(mungefilename ${filename:t})
+  # if [[ "${filename}" != "${newFilename}" ]]; then
+  #     mv -v ${filename} ${newFilename} && filename=${newFilename}
+  # fi
+
+  if [[ -n "${filename}" && -e "${filename}" && ${ytResult} -eq 0 ]]; then
+      max_vol=$( \
+          pv -N "$(trimName "vol:" ${title})" ${filename} | ffmpeg -i pipe:0 -af "volumedetect" -vn -f null /dev/null 2>&1 | \
+              grep 'max_volume' | sed -r 's/^.*max_volume: ([-0-9\.]+) dB$/\1/'
+             )
+      printf "Maximum volume: %.2fdB\n" "${max_vol}"
+      [[ -n "${max_vol}" ]] || max_vol=0
+      bare_amp=$(printf "%.2f" $(($max_vol * (-1))))
+      amp=$(printf "volume=%sdB" "${bare_amp}")
+
+      typeset -a extensions
+      extensions=("mkv" "mp3" "mp4" "webm")
+
+      if [[ ${msgTags[(i)tv]} -le $#msgTags ]]; then
+          extensions=("mp4")
+      fi
+
+      if [[ "${bare_amp}" -ne 0 || ${extensions[(i)${filename:e}]} -gt ${#extensions} ]]; then
+          printf "Transcoding ‘%s’" ${title} | debug
+          printf "%d %d %d/%d(%d)…\n" \
+                 $([[ $($alwaysTranscode; print $?) -eq 0 ]]; print $?) \
+                 $([[ "${bare_amp}" -ne 0 ]]; print $?) \
+                 ${extensions[(i)${filename:e}]} ${#extensions} \
+                 $([[ ${extensions[(i)${filename:e}]} -gt ${#extensions} ]]; print $?) \
+              | debug
+          tempfile=$(mktemp --tmpdir=${filename:h} .transcode.${filename:t:r}.$$.XXXXXX.${filename:e})
+          cleanup rm -v -- "${tempfile}"
+          mv -vf "${filename}" "${tempfile}" | debug
+
+          typeset -A fileInfo
+          for line ($(ffprobe -v error -show_format -show_streams -show_entries stream=codec_name,codec_type:format=:stream_tags=:stream_disposition=:format_tags= -of flat=h=0 -- ${tempfile})); do
+              fileInfo[${line%=*}]=${(Q)line#*=}
+          done
+
+          ext=${extensions[1]}
+          filename=${filename%.*}.${ext}
+          typeset -a args
+          args=(-y)
+          typeset -a prePass
+          prePass=()
+          typeset -a cargs
+          cargs=()
+          typeset -a p1args
+          p1args=()
+          typeset -a p2args
+          p2args=()
+          for formatKey (${(k)fileInfo}); do
+              [[ ${formatKey} =~ "stream.([0-9]+).codec_type" ]] || continue
+              local i=$match[1]
+              local cType=${fileInfo[${formatKey}]}
+              local cName=${fileInfo[stream.${i}.codec_name]}
+
+              printf "Stream %d: %s (%s)\n" ${i} ${cName} ${cType} | debug
+              
+              if [[ ${cType} == "video" ]]; then
+                  if [[ ${ext} == "mkv" ]]; then
+                      case ${cName} in
+                          h264|vp9|png)
+                              cargs+=(-c:${i} copy)
+                              ;;
+                          *)
+                              # p1args=(-pass 1 -threads 8 -speed 4 -tile-columns 6 -frame-parallel 1)
+                              # prePass+=(-c:${i} libvpx-vp9 -b:${i} 0 -crf:${i} 33)
+                              # p2args+=(-pass 2 -threads 8 -speed 2 -tile-columns 6 -frame-parallel 1 -auto-alt-ref 1 -lag-in-frames 25)
+                              # cargs+=(-c:${i} libvpx-vp9 -b:${i} 0 -crf:${i} 33)
+                              cargs+=(-c:${i} libvpx-vp9 -b:${i} 2M -threads 8 -tile-columns 6 -frame-parallel 1)
+                              ;;
+                      esac
+                  elif [[ ${ext} == "mp4" ]]; then
+                      case ${cName} in
+                          mpeg|h264)
+                              cargs+=(-c:${i} copy)
+                              ;;
+                          *)
+                              p2args+=(-strict -2)
+                              cargs+=(-c:${i} libx264)
+                              ;;
+                      esac
+                  fi
+              elif [[ ${cType} == "audio" ]]; then
+                  if [[ ${ext} == "mkv" ]]; then
+                      if [[ ( ${cName} == opus || ${cName} == flac || ${cName} == vorbis ) && "${bare_amp}" -eq 0 ]]; then
+                          cargs+=(-c:${i} copy)
+                      else
+                          p2args+=(-vbr on -compression_level 10)
+                          cargs+=(-c:${i} libopus -b:${i} 256K)
+                      fi
+                  elif [[ ${ext} == "mp4" ]]; then
+                      if [[ ${cName} == aac && "${bare_amp}" -eq 0 ]]; then
+                          cargs+=(-c:${i} copy)
+                      else
+                          cargs+=(-c:${i} aac)
+                      fi
+                  fi
+              fi
+          done
+
+          [[ "${bare_amp}" -ne 0 ]] && p2args+=(-af "${amp}")
+
+          if [[ $#prePass -gt 0 ]]; then
+              args+=(-v info -i ${tempfile})
+
+              oldPwd=${PWD}
+              cd $(mktemp -d --tmpdir "transcode.${0:t}.$$.XXXXXX")
+              cleanup rm -rfv -- ${PWD}
+
+              p1args+=(-an -f matroska)
+              print -- ${prePass} ${p1args} | debug
+              ffmpeg ${args} ${prePass} ${p1args} -- /dev/null
+
+              print -- ${cargs} ${p2args} | debug
+              ffmpeg ${args} ${cargs} ${p2args} -- "${filename}"
+
+              cd ${oldPwd}
+          else
+              args+=(-v warning -i pipe:0)
+
+              print -- ${cargs} ${p2args} | debug
+              pv -N "$(trimName "trans:" ${title})" ${tempfile} | ffmpeg ${args} ${cargs} ${p2args} -- "${filename}"
+          fi
+      fi
+      
+      chmod -v 644 "${filename}" | debug
+      tmpFile=$(mktemp --tmpdir=/home/gkleen/rss/tmp .insertUrl.$$.XXXXXX)
+      relUrl=$(realpath --relative-to=/srv/media ${filename})
+      typeset -a relUrlComponents
+      relUrlComponents=(${(s./.)relUrl})
+      relUrl=""
+      for urlPiece (${relUrlComponents}); do
+          [[ -n "${relUrl}" ]] && relUrl+="/"
+          relUrl+=$(sed -f ${HOME}/url_escape.sed <<<${urlPiece})
+      done
+      awk -v "link=http://odin.asgard.yggdrasil/${relUrl}" '{ if (r == 0) { r = gsub("href=\"[^\"]+\"", "href=\"" link "\""); }; print; }' "${message}" >! $tmpFile
+      mv -v $tmpFile $message | debug
+      notmuch tag '+cached' -- $msgId && printf "Tagged ‘%s’ as 'cached'" ${msgId}
+      
+      if [[ -n "$(notmuch search "tag:inbox AND $msgId")" ]]; then
+          if [[ ${msgTags[(i)tv]} -le $#msgTags ]]; then
+              printf "%s\n%s\n" "Media available on odin" "${title}" \
+                  | uux -p -n 'hel!notify-gkleen' -a download_youtube -u normal \
+                  && print "Sent notification to hel via uucp"
+          else
+              queue.hel "${filename}" && notmuch tag '-inbox' '-unread' -- $msgId
+          fi
+      else
+          print "Message vanished from inbox" | warn
+      fi
+  else
+      printf "An error occured while downloading video at ‘%s’ (exitcode: %d)\n" ${url} ${ytResult} >&2
+      # notmuch tag +failed -- ${msgId}
+      # exit $?
+  fi
+) 9<>"${message}"
diff --git a/users/gkleen@odin.nix b/users/gkleen@odin.nix
index 2c63c085..d06215d1 100644
--- a/users/gkleen@odin.nix
+++ b/users/gkleen@odin.nix
@@ -1,2 +1,5 @@
 {
+  packageOverrides = pkgs: with pkgs; {
+    strm = pkgs.callPackage ../odin/strm.nix {};
+  };
 }