path: root/odin/strm/download_youtube

#!@zsh@/bin/zsh

alwaysTranscode=false

ffmpeg() { { { @ffmpeg@/bin/ffmpeg $@ 1>&3 } 2>&1 | stdbuf -o 0 tr '\r' '\n' | grep -v --line-buffered -E '^$' 1>&2 } 3>&1 }
pv() { { { @pv@/bin/pv -D 2 -i 2 -w 100 -H 1 -f $@ 1>&3 } 2>&1 | stdbuf -o 0 tr '\r' '\n' | grep -v --line-buffered -E '^$' 1>&2 } 3>&1 }
trimName() { prefix=$1; shift; printf "%s%s" ${prefix} $(awk -v len=$((19 - $#prefix)) '{ if (length($0) > len) print substr($0, 1, len-3) "..."; else print; }' <<<"${(j: :)@}")}

function notmuch {
    while true; do 
        result=$(env NOTMUCH_CONFIG=${HOME}/.notmuch-rss-config @notmuch@/bin/notmuch "$@" 2>&1)
        if ! [[ $result =~ "already locked" ]]; then
            echo -nE $result
            return
        fi
        sleep 2
    done
}

logTag=${0:t}

exec 1> >(logger -t "$logTag" -p news.notice)
exec 2> >(logger -t "$logTag" -p news.error)

debug() { logger -t "$logTag" -p news.debug }
warn() { logger -t "$logTag" -p news.warn }

typeset -a cleanupCmds
cleanupCmds=()

function doCleanup {
    for cmd (${cleanupCmds}); do
        # print -- ${cmd} | debug
        eval $cmd | debug
    done
}

function cleanup() {
    local cmd
    cmd=""
    for arg ($@); do
        [[ -n ${cmd} ]] && cmd="$cmd "
        cmd="${cmd}${(qq)arg}"
    done

    cleanupCmds+=(${cmd})
}

mungefilename ()
{
    tr "\`\"' /[:upper:]" "_____[:lower:]" <<<${@} | sed -r 's/[^0-9a-z\!\#\$\%\&\(\)\*\+\,\-\.\:\;\<\=\>\?\@\[\]\~\^\_\{\}\|]//g; s/_+/_/g'
}

[[ $#@ -le 0 || $#@ -gt 1 ]] && exit 2

msgId=$1
message=$(notmuch search --output=files --duplicate=1 $msgId)
[[ ! -e $message ]] && exit 1

printf ">>> %s <<<\n  %s\n" "${msgId}" "${message}"

( print "  waiting for lock..."
  
  if ! flock -xn 9; then
      print "  could not acquire lock."
      exit 0
  fi

  print "  locked."

  trap doCleanup EXIT

  typeset -a msgTags
  msgTags=($(notmuch search --output=tags $msgId ))

  if [[ ${msgTags[(i)cached]} -le $#msgTags ]]; then
      print "Message in Cache" | warn
      exit 0
  fi

  if [[ 'base64' == $(sed '/^Content-Transfer-Encoding: */!d; s///;q' ${message}) ]]; then
      tmpFile=$(mktemp --tmpdir=/home/gkleen/rss/tmp .writeOut.XXXXXX)
      printf "Decoding base64 message content for ‘%s’" ${message} | debug
      sed '/^Content-Transfer-Encoding: */d; /^$/q' ${message} >! $tmpFile
      sed '1,/^$/d' ${message} | base64 -d >> $tmpFile
      mv -v $tmpFile ${message}
  fi

  from=$(mungefilename $(awk '/^From/ { gsub("^\"", "", $2); print $2; exit; }' "${message}"))
  if grep -q "<p>Enclosure: <a" "${message}"; then
      url=$(tr -d '\n' < "${message}" | sed -r 's/^.*<p>Enclosure: <a[^>]+href=[^">]*"([^"]+)".*$/\1/')
  else
      url=$(awk '/^X-RSS-URL/ { print $2; exit; }' "${message}")
  fi

  sld=$(cut -d '/' -f 3 <<<${url} | rev | cut -d '.' -f 1-2 | rev)

  if [[ -e ${HOME}/.dl-backup ]] && grep -q ${sld} ${HOME}/.dl-backup; then
      printf "Was told to back up off %s\n" ${sld} | warn
      exit 0
  else
      printf "Proceeding on %s\n" ${sld} | debug
  fi

  dir="/srv/media/youtube"
  if ! { [[ -d $dir ]] || mkdir -p $dir }; then
      exit 1
  fi

  formatString='bestvideo[width<=2560][height<=1440][fps<=60][vcodec=vp9]+bestaudio[acodec=opus]/bestvideo[width<=2560][height<=1440][fps<=60]+bestaudio/best[width<=2560][height<=1440][fps<=60]/best'

  if [[ ${msgTags[(i)tv]} -le $#msgTags ]]; then
      formatString="bestvideo[width<=2560][height<=1440][fps<=60][vcodec=h264]+bestaudio[acodec=aac]/bestvideo[width<=2560][height<=1440][fps<=60][vcodec=mpeg]+bestaudio[acodec=aac]/mp4[width<=2560][height<=1440][fps<=60][vcodec=h264][acodec=aac]/mp4[width<=2560][height<=1440][fps<=60][vcodec=mpeg][acodec=aac]/${formatString}"
  fi

  typeset -a args
  args=(--no-playlist --add-metadata --newline --mark-watched -f ${formatString})

  @youtubedl@/bin/youtube-dl ${args} --get-filename -o $'%(extractor)s\n%(id)s\n%(format)s\n%(title)s\n%(ext)s' -- ${url} | {
      oldIFS=${IFS}
      export IFS=

      read extractor
      read videoId
      read format
      read title
      read fileExtension

      export IFS=${oldIFS}
  } || {
      ret=$?
      printf "An error occured while determining video metadata (exitcode: %d)\n" ${ret} >&2
      notmuch tag +failed -- ${msgId}
      exit $?
  }

  if [[ "${extractor}" == "generic" ]]; then
      title=$(grep -E '^Subject' "${message}" | head -n 1 | cut -d ' ' -f 2-)
  fi

  filename="${dir}/${title}-${extractor}.${videoId}.${from}.${fileExtension}"
  filename=$(tr $'\n\0\/' ' ' <<<${filename} | tr -d $'!') 

  printf "%s\n%s via %s: %s\nExpecting ‘%s’\n" ${title} ${videoId} ${extractor} ${format} ${filename:t}

  if [[ -n ${extractor} && -n ${videoId} ]]; then
      printf "Searching for %s on %s in cache...\n" ${videoId} ${extractor} | debug

      typeset -a messages
      messages=($(notmuch search --output=messages -- from:${extractor} and ${videoId} and is:cached))

      if [[ ${#messages} -ne 0 ]]; then
          printf "Video already cached: %s\n" "${messages[*]}" | warn
      else
          printf "Video not yet cached\n" | debug
      fi
  fi

  @youtubedl@/bin/youtube-dl $args -o "${${filename:r}//\%/%%}.%(ext)s" -- ${url} 2>&1 </dev/null | stdbuf -o0 tr '\r' '\n'
  ytResult=$?

  if [[ ! -e ${filename} && -n ${filename} ]]; then
      for f (${filename:r}.*(N)); do
          shouldDelete=false

          grep -qE '\.f[0-9]+\.' <<<${f} && shouldDelete=true
          grep -qE '\.part(-Frag[0-9]+)?$' <<<${f} && shouldDelete=true
          grep -qE '\.temp\.[^\.]+$' <<<${f} && shouldDelete=true

          if ${shouldDelete}; then
              rm -v ${f}
          else
              filename=${f}
          fi 
      done
  fi

  [[ -n ${filename} ]] && printf "Found ‘%s’\n" ${filename:t} || debug

  # newFilename=${filename:h}/$(mungefilename ${filename:t})
  # if [[ "${filename}" != "${newFilename}" ]]; then
  #     mv -v ${filename} ${newFilename} && filename=${newFilename}
  # fi

  if [[ -n "${filename}" && -e "${filename}" && ${ytResult} -eq 0 ]]; then
      max_vol=$( \
          pv -N "$(trimName "vol:" ${title})" ${filename} | ffmpeg -i pipe:0 -af "volumedetect" -vn -f null /dev/null 2>&1 | \
              grep 'max_volume' | sed -r 's/^.*max_volume: ([-0-9\.]+) dB$/\1/'
             )
      printf "Maximum volume: %.2fdB\n" "${max_vol}"
      [[ -n "${max_vol}" ]] || max_vol=0
      bare_amp=$(printf "%.2f" $(($max_vol * (-1))))
      amp=$(printf "volume=%sdB" "${bare_amp}")

      typeset -a extensions
      extensions=("mkv" "mp3" "mp4" "webm")

      if [[ ${msgTags[(i)tv]} -le $#msgTags ]]; then
          extensions=("mp4")
      fi

      if [[ "${bare_amp}" -ne 0 || ${extensions[(i)${filename:e}]} -gt ${#extensions} ]]; then
          printf "Transcoding ‘%s’" ${title} | debug
          printf "%d %d %d/%d(%d)…\n" \
                 $([[ $($alwaysTranscode; print $?) -eq 0 ]]; print $?) \
                 $([[ "${bare_amp}" -ne 0 ]]; print $?) \
                 ${extensions[(i)${filename:e}]} ${#extensions} \
                 $([[ ${extensions[(i)${filename:e}]} -gt ${#extensions} ]]; print $?) \
              | debug
          tempfile=$(mktemp --tmpdir=${filename:h} .transcode.${filename:t:r}.$$.XXXXXX.${filename:e})
          cleanup rm -v -- "${tempfile}"
          mv -vf "${filename}" "${tempfile}" | debug

          typeset -A fileInfo
          for line ($(@ffmpeg@/bin/ffprobe -v error -show_format -show_streams -show_entries stream=codec_name,codec_type:format=:stream_tags=:stream_disposition=:format_tags= -of flat=h=0 -- ${tempfile})); do
              fileInfo[${line%=*}]=${(Q)line#*=}
          done

          ext=${extensions[1]}
          filename=${filename%.*}.${ext}
          typeset -a args
          args=(-y)
          typeset -a prePass
          prePass=()
          typeset -a cargs
          cargs=()
          typeset -a p1args
          p1args=()
          typeset -a p2args
          p2args=()
          for formatKey (${(k)fileInfo}); do
              [[ ${formatKey} =~ "stream.([0-9]+).codec_type" ]] || continue
              local i=$match[1]
              local cType=${fileInfo[${formatKey}]}
              local cName=${fileInfo[stream.${i}.codec_name]}

              printf "Stream %d: %s (%s)\n" ${i} ${cName} ${cType} | debug
              
              if [[ ${cType} == "video" ]]; then
                  if [[ ${ext} == "mkv" ]]; then
                      case ${cName} in
                          h264|vp9|png)
                              cargs+=(-c:${i} copy)
                              ;;
                          *)
                              # p1args=(-pass 1 -threads 8 -speed 4 -tile-columns 6 -frame-parallel 1)
                              # prePass+=(-c:${i} libvpx-vp9 -b:${i} 0 -crf:${i} 33)
                              # p2args+=(-pass 2 -threads 8 -speed 2 -tile-columns 6 -frame-parallel 1 -auto-alt-ref 1 -lag-in-frames 25)
                              # cargs+=(-c:${i} libvpx-vp9 -b:${i} 0 -crf:${i} 33)
                              cargs+=(-c:${i} libvpx-vp9 -b:${i} 2M -threads 8 -tile-columns 6 -frame-parallel 1)
                              ;;
                      esac
                  elif [[ ${ext} == "mp4" ]]; then
                      case ${cName} in
                          mpeg|h264)
                              cargs+=(-c:${i} copy)
                              ;;
                          *)
                              p2args+=(-strict -2)
                              cargs+=(-c:${i} libx264)
                              ;;
                      esac
                  fi
              elif [[ ${cType} == "audio" ]]; then
                  if [[ ${ext} == "mkv" ]]; then
                      if [[ ( ${cName} == opus || ${cName} == flac || ${cName} == vorbis ) && "${bare_amp}" -eq 0 ]]; then
                          cargs+=(-c:${i} copy)
                      else
                          p2args+=(-vbr on -compression_level 10)
                          cargs+=(-c:${i} libopus -b:${i} 256K)
                      fi
                  elif [[ ${ext} == "mp4" ]]; then
                      if [[ ${cName} == aac && "${bare_amp}" -eq 0 ]]; then
                          cargs+=(-c:${i} copy)
                      else
                          cargs+=(-c:${i} aac)
                      fi
                  fi
              fi
          done

          [[ "${bare_amp}" -ne 0 ]] && p2args+=(-af "${amp}")

          if [[ $#prePass -gt 0 ]]; then
              args+=(-v info -i ${tempfile})

              oldPwd=${PWD}
              cd $(mktemp -d --tmpdir "transcode.${0:t}.$$.XXXXXX")
              cleanup rm -rfv -- ${PWD}

              p1args+=(-an -f matroska)
              print -- ${prePass} ${p1args} | debug
              ffmpeg ${args} ${prePass} ${p1args} -- /dev/null

              print -- ${cargs} ${p2args} | debug
              ffmpeg ${args} ${cargs} ${p2args} -- "${filename}"

              cd ${oldPwd}
          else
              args+=(-v warning -i pipe:0)

              print -- ${cargs} ${p2args} | debug
              pv -N "$(trimName "trans:" ${title})" ${tempfile} | ffmpeg ${args} ${cargs} ${p2args} -- "${filename}"
          fi
      fi
      
      chmod -v 644 "${filename}" | debug
      tmpFile=$(mktemp --tmpdir=/home/gkleen/rss/tmp .insertUrl.$$.XXXXXX)
      relUrl=$(realpath --relative-to=/srv/media ${filename})
      typeset -a relUrlComponents
      relUrlComponents=(${(s./.)relUrl})
      relUrl=""
      for urlPiece (${relUrlComponents}); do
          [[ -n "${relUrl}" ]] && relUrl+="/"
          relUrl+=$(sed -f ${HOME}/url_escape.sed <<<${urlPiece})
      done
      awk -v "link=http://odin.asgard.yggdrasil/${relUrl}" '{ if (r == 0) { r = gsub("href=\"[^\"]+\"", "href=\"" link "\""); }; print; }' "${message}" >! $tmpFile
      mv -v $tmpFile $message | debug
      notmuch tag '+cached' -- $msgId && printf "Tagged ‘%s’ as 'cached'" ${msgId}
      
      if [[ -n "$(notmuch search "tag:inbox AND $msgId")" ]]; then
          if [[ ${msgTags[(i)tv]} -le $#msgTags ]]; then
              printf "%s\n%s\n" "Media available on odin" "${title}" \
                  | uux -r -p -n 'hel!notify-gkleen' -a download_youtube -u normal \
                  && print "Sent notification to hel via uucp"
          else
              queue.hel -r "${filename}" && notmuch tag '-inbox' '-unread' -- $msgId
          fi
          systemctl --no-block start uucico@hel.service
      else
          print "Message vanished from inbox" | warn
      fi
  else
      printf "An error occured while downloading video at ‘%s’ (exitcode: %d)\n" ${url} ${ytResult} >&2
      # notmuch tag +failed -- ${msgId}
      # exit $?
  fi
) 9<>"${message}"