path: root/odin/strm/download_youtube

#!@zsh@/bin/zsh

alwaysTranscode=false

ffmpeg() { { { @ffmpeg@/bin/ffmpeg $@ 1>&3 } 2>&1 | stdbuf -o 0 tr '\r' '\n' | grep -v --line-buffered -E '^$' 1>&2 } 3>&1 }
pv() { { { @pv@/bin/pv -D 2 -i 2 -w 100 -H 1 -f $@ 1>&3 } 2>&1 | stdbuf -o 0 tr '\r' '\n' | grep -v --line-buffered -E '^$' 1>&2 } 3>&1 }
trimName() { prefix=$1; shift; printf "%s%s" ${prefix} $(awk -v len=$((19 - $#prefix)) '{ if (length($0) > len) print substr($0, 1, len-3) "..."; else print; }' <<<"${(j: :)@}")}

function notmuch {
    while true; do 
        result=$(env NOTMUCH_CONFIG=${HOME}/.notmuch-rss-config @notmuch@/bin/notmuch "$@" 2>&1)
        if ! [[ $result =~ "already locked" ]]; then
            echo -nE $result
            return
        fi
        sleep 2
    done
}

typeset -a cleanupCmds
cleanupCmds=()

function doCleanup {
    for cmd (${cleanupCmds}); do
        # print -- ${cmd}
        eval $cmd
    done
}

function cleanup() {
    local cmd
    cmd=""
    for arg ($@); do
        [[ -n ${cmd} ]] && cmd="$cmd "
        cmd="${cmd}${(qq)arg}"
    done

    cleanupCmds+=(${cmd})
}

mungefilename ()
{
    tr "\`\"' /[:upper:]" "_____[:lower:]" <<<${@} | sed -r 's/[^0-9a-z\!\#\$\%\&\(\)\*\+\,\-\.\:\;\<\=\>\?\@\[\]\~\^\_\{\}\|]//g; s/_+/_/g'
}

[[ $#@ -le 0 || $#@ -gt 1 ]] && exit 2

msgId=$1
message=$(notmuch search --output=files --duplicate=1 $msgId)
[[ ! -e $message ]] && exit 1

printf ">>> %s <<<\n  %s\n" "${msgId}" "${message}"

( print "  waiting for lock..."
  
  if ! flock -xn 9; then
      print "  could not acquire lock."
      exit 0
  fi

  print "  locked."

  trap doCleanup EXIT

  typeset -a msgTags
  msgTags=($(notmuch search --output=tags $msgId ))

  if [[ ${msgTags[(i)cached]} -le $#msgTags ]]; then
      print "Message in Cache" >&2
      exit 0
  fi

  if [[ 'base64' == $(sed '/^Content-Transfer-Encoding: */!d; s///;q' ${message}) ]]; then
      tmpFile=$(mktemp --tmpdir=/home/gkleen/rss/tmp .writeOut.XXXXXX)
      printf "Decoding base64 message content for ‘%s’\n" ${message}
      sed '/^Content-Transfer-Encoding: */d; /^$/q' ${message} >! $tmpFile
      sed '1,/^$/d' ${message} | @coreutils@/bin/base64 -d >> $tmpFile
      mv -v $tmpFile ${message}
  fi
  
  if [[ 'quoted-printable' == $(sed '/^Content-Transfer-Encoding: */!d; s///;q' ${message}) ]]; then
      tmpFile=$(mktemp --tmpdir=/home/gkleen/rss/tmp .writeOut.XXXXXX)
      printf "Decoding quoted printable message content for ‘%s’\n" ${message}
      sed '/^Content-Transfer-Encoding: */d; /^$/q' ${message} >! $tmpFile
      sed '1,/^$/d' ${message} | @qprint@/bin/qprint -d >> $tmpFile
      mv -v $tmpFile ${message}
  fi


  from=$(mungefilename $(awk '/^From/ { gsub("^\"", "", $2); print $2; exit; }' "${message}"))
  if grep -q "<p>Enclosure: <a" "${message}"; then
      url=$(tr -dC '[:graph:][:blank:]' < "${message}" | sed -r 's/^.*<p>Enclosure: <a[^>]+href=[^">]*"([^"]+)".*$/\1/')
  else
      url=$(awk '/^X-RSS-URL/ { print $2; exit; }' "${message}")
  fi

  sld=$(cut -d '/' -f 3 <<<${url} | rev | cut -d '.' -f 1-2 | rev)

  if [[ -e ${HOME}/.dl-backup ]] && grep -q ${sld} ${HOME}/.dl-backup; then
      printf "Was told to back up off %s\n" ${sld} >&2
      exit 0
  else
      printf "Proceeding on %s\n" ${sld}
  fi

  dir="/srv/media/youtube"
  if ! { [[ -d $dir ]] || mkdir -p $dir }; then
      exit 1
  fi

  formatString='bestvideo[width<=3840][height<=2160][fps<=60][vcodec=vp9]+bestaudio[acodec=opus]/bestvideo[width<=3840][height<=2160][fps<=60]+bestaudio/best[width<=3840][height<=2160][fps<=60]/best'

  if [[ ${msgTags[(i)tv]} -le $#msgTags ]]; then
      formatString="bestvideo[width<=2560][height<=1440][fps<=60][vcodec=h264]+bestaudio[acodec=aac]/bestvideo[width<=2560][height<=1440][fps<=60][vcodec=mpeg]+bestaudio[acodec=aac]/mp4[width<=2560][height<=1440][fps<=60][vcodec=h264][acodec=aac]/mp4[width<=2560][height<=1440][fps<=60][vcodec=mpeg][acodec=aac]/${formatString}"
  fi

  typeset -a args
  args=(--no-playlist --add-metadata --newline --mark-watched -f ${formatString} --match-filter '!is_live')

  @youtubedl@/bin/youtube-dl ${args} --get-filename -o $'%(extractor)s\n%(id)s\n%(format)s\n%(title)s\n%(ext)s' -- ${url} | {
      oldIFS=${IFS}
      export IFS=

      read extractor
      read videoId
      read format
      read title
      read fileExtension

      export IFS=${oldIFS}
  } || {
      ret=$?
      printf "An error occured while determining video metadata (exitcode: %d)\n" ${ret} >&2
      notmuch tag +failed -- ${msgId}
      exit $?
  }

  if [[ "${extractor}" == "generic" ]]; then
      title=$(grep -E '^Subject' "${message}" | head -n 1 | cut -d ' ' -f 2-)
  fi

  filename="${title}-${extractor}.${videoId}.${from}.${fileExtension}"
  filename=$(tr $'\n\0\/' ' ' <<<${filename} | tr -d $'!' | awk '{$1=$1};1') 
  filename="${dir}/${filename}"

  printf "%s\n%s via %s: %s\nExpecting ‘%s’\n" ${title} ${videoId} ${extractor} ${format} ${filename:t}

  if [[ -n ${extractor} && -n ${videoId} ]]; then
      printf "Searching for %s on %s in cache...\n" ${videoId} ${extractor}

      typeset -a messages
      messages=($(notmuch search --output=messages -- from:${extractor} and ${videoId} and is:cached))

      if [[ ${#messages} -ne 0 ]]; then
          printf "Video already cached: %s\n" "${messages[*]}" >&2
      else
          printf "Video not yet cached\n"
      fi
  fi

  @youtubedl@/bin/youtube-dl $args -o "${${filename:r}//\%/%%}.%(ext)s" -- ${url} 2>&1 </dev/null | stdbuf -o0 tr '\r' '\n'
  ytResult=$?

  if [[ ! -e ${filename} && -n ${filename} ]]; then
      for f (${filename:r}.*(N)); do
          shouldDelete=false

          grep -qE '\.f[0-9]+\.' <<<${f} && shouldDelete=true
          grep -qE '\.part(-Frag[0-9]+)?$' <<<${f} && shouldDelete=true
          grep -qE '\.temp\.[^\.]+$' <<<${f} && shouldDelete=true

          if ${shouldDelete}; then
              rm -v ${f}
          else
              filename=${f}
          fi 
      done
  fi

  [[ -n ${filename} ]] && printf "Found ‘%s’\n" ${filename:t}

  # newFilename=${filename:h}/$(mungefilename ${filename:t})
  # if [[ "${filename}" != "${newFilename}" ]]; then
  #     mv -v ${filename} ${newFilename} && filename=${newFilename}
  # fi

  if [[ -n "${filename}" && -e "${filename}" && ${ytResult} -eq 0 ]]; then
    max_vol=$( \
      pv -N "$(trimName "vol:" ${title})" ${filename} | ffmpeg -i pipe:0 -af "volumedetect" -vn -f null /dev/null 2>&1 | \
      grep 'max_volume' | sed -r 's/^.*max_volume: ([-0-9\.]+) dB$/\1/'
    )
    printf "Maximum volume: %.2fdB\n" "${max_vol}"
    [[ -n "${max_vol}" ]] || max_vol=0
    bare_amp=$(printf "%.2f" $(($max_vol * (-1))))
    amp=$(printf "volume=%sdB" "${bare_amp}")

    typeset -a extensions
    extensions=("mkv" "mp3" "mp4" "webm")

    if [[ ${msgTags[(i)tv]} -le $#msgTags ]]; then
        extensions=("mp4")
    fi

    typeset -a transcodeExtractors
    transcodeExtractors=()

    if [[ "${bare_amp}" -ne 0
       || ${extensions[(i)${filename:e}]} -gt ${#extensions}
       || ${transcodeExtractors[(i)${extractor}]} -le ${#transcodeExtractors}
       ]]; then
      printf "Transcoding ‘%s’\n" ${title}
      printf "%d %d %d/%d(%d)…\n" \
             $([[ $($alwaysTranscode; print $?) -eq 0 ]]; print $?) \
             $([[ "${bare_amp}" -ne 0 ]]; print $?) \
             ${extensions[(i)${filename:e}]} ${#extensions} \
             $([[ ${extensions[(i)${filename:e}]} -gt ${#extensions} ]]; print $?)
      tempfile=$(mktemp --tmpdir=${filename:h} .transcode.${filename:t:r}.$$.XXXXXX.${filename:e})
      cleanup rm -v -- "${tempfile}"
      mv -vf "${filename}" "${tempfile}"

      typeset -A fileInfo
      for line ($(@ffmpeg@/bin/ffprobe -v error -show_format -show_streams -show_entries stream=codec_name,codec_type:format=:stream_tags=:stream_disposition=:format_tags= -of flat=h=0 -- ${tempfile})); do
          fileInfo[${line%=*}]=${(Q)line#*=}
      done

      ext=${extensions[1]}
      filename=${filename%.*}.${ext}

      typeset -a transModes
      transModes=(use-index enumerate)

      for transMode (${transModes}); do
        printf "Trying transMode=%s\n" ${transMode}
          
        typeset -a args
        args=(-y)
        typeset -a prePass
        prePass=()
        typeset -a cargs
        cargs=()
        typeset -a p1args
        p1args=()
        typeset -a p2args
        p2args=()

        local j=0
        
        for formatKey (${(k)fileInfo}); do
            [[ ${formatKey} =~ "stream.([0-9]+).codec_type" ]] || continue
            local i=$match[1]
            local cType=${fileInfo[${formatKey}]}
            local cName=${fileInfo[stream.${i}.codec_name]}

            if [[ ${transMode} == "enumerate" ]]; then
                i=${j}
                j=$((j + 1))
            fi

            printf "Stream %d: %s (%s)\n" ${i} ${cName} ${cType}
            
            if [[ ${cType} == "video" ]]; then
                if [[ ${ext} == "mkv" ]]; then
                    case ${cName} in
                        h264|vp9|png)
                            cargs+=(-c:${i} copy)
                            ;;
                        *)
                            # p1args=(-pass 1 -threads 8 -speed 4 -tile-columns 6 -frame-parallel 1)
                            # prePass+=(-c:${i} libvpx-vp9 -b:${i} 0 -crf:${i} 33)
                            # p2args+=(-pass 2 -threads 8 -speed 2 -tile-columns 6 -frame-parallel 1 -auto-alt-ref 1 -lag-in-frames 25)
                            # cargs+=(-c:${i} libvpx-vp9 -b:${i} 0 -crf:${i} 33)
                            cargs+=(-c:${i} libvpx-vp9 -b:${i} 2M -threads 8 -tile-columns 6 -frame-parallel 1)
                            ;;
                    esac
                elif [[ ${ext} == "mp4" ]]; then
                    case ${cName} in
                        mpeg|h264)
                            cargs+=(-c:${i} copy)
                            ;;
                        *)
                            p2args+=(-strict -2)
                            cargs+=(-c:${i} libx264)
                            ;;
                    esac
                fi
            elif [[ ${cType} == "audio" ]]; then
                if [[ ${ext} == "mkv" ]]; then
                    if [[ ( ${cName} == opus || ${cName} == flac || ${cName} == vorbis ) && "${bare_amp}" -eq 0 ]]; then
                        cargs+=(-c:${i} copy)
                    else
                        p2args+=(-vbr on -compression_level 10)
                        cargs+=(-c:${i} libopus -b:${i} 256K)
                    fi
                elif [[ ${ext} == "mp4" ]]; then
                    if [[ ${cName} == aac && "${bare_amp}" -eq 0 ]]; then
                        cargs+=(-c:${i} copy)
                    else
                        cargs+=(-c:${i} aac)
                    fi
                fi
            fi
        done
        
        [[ "${bare_amp}" -ne 0 ]] && p2args+=(-af "${amp}")

        transcodeStdout=$(mktemp --tmpdir "transcode.stdout.${0:t}.$$.XXXXXX")
        cleanup rm -rfv -- ${transcodeStdout}
        transcodeStderr=$(mktemp --tmpdir "transcode.stderr.${0:t}.$$.XXXXXX")
        cleanup rm -rfv -- ${transcodeStderr}

        (
            if [[ $#prePass -gt 0 ]]; then
                args+=(-v info -i ${tempfile})
                
                oldPwd=${PWD}
                cd $(mktemp -d --tmpdir "transcode.${0:t}.$$.XXXXXX")
                cleanup rm -rfv -- ${PWD}
                
                p1args+=(-an -f matroska)
                print -- ${prePass} ${p1args}
                ffmpeg ${args} ${prePass} ${p1args} -- /dev/null
                
                print -- ${cargs} ${p2args}
                ffmpeg ${args} ${cargs} ${p2args} -- "${filename}"
                
                cd ${oldPwd}
            else
                args+=(-v warning -i pipe:0)
                
                print -- ${cargs} ${p2args}
                pv -N "$(trimName "trans:" ${title})" ${tempfile} | ffmpeg ${args} ${cargs} ${p2args} -- "${filename}"
            fi
        ) >&1 >${transcodeStdout} 2>&2 2>${transcodeStderr}

        if ! grep -q "Invalid encoder type" ${transcodeStdout} ${transcodeStderr}; then
            printf "Successfully transcoded using transMode=%s\n" ${transMode}
            break
        fi
      done
    fi
  fi
      
  if [[ -n "${filename}" && -e "${filename}" && ${ytResult} -eq 0 ]]; then
      chmod -v 644 "${filename}"
      tmpFile=$(mktemp --tmpdir=/home/gkleen/rss/tmp .insertUrl.$$.XXXXXX)
      relUrl=$(realpath --relative-to=/srv/media ${filename})
      typeset -a relUrlComponents
      relUrlComponents=(${(s./.)relUrl})
      relUrl=""
      for urlPiece (${relUrlComponents}); do
          [[ -n "${relUrl}" ]] && relUrl+="/"
          relUrl+=$(sed -f ${HOME}/url_escape.sed <<<${urlPiece})
      done
      awk -v "link=http://odin.asgard.yggdrasil/${relUrl}" '{ if (r == 0) { r = gsub("href=\"[^\"]+\"", "href=\"" link "\""); }; print; }' "${message}" >! $tmpFile
      mv -v $tmpFile $message
      notmuch tag '+cached' -- $msgId && printf "Tagged ‘%s’ as 'cached'\n" ${msgId}
      
      if [[ -n "$(notmuch search "tag:inbox AND $msgId")" ]]; then
          if [[ ${msgTags[(i)tv]} -le $#msgTags ]]; then
              printf "%s\n%s\n" "Media available on odin" "${title}" \
                  | uux -r -p -n '@remoteNode@!notify-gkleen' -a download_youtube -u normal \
                  && print "Sent notification to @remoteNode@ via uucp"
          else
              queue.@remoteNode@ -r "${filename}" && notmuch tag '-inbox' '-unread' -- $msgId
          fi
      else
          print "Message vanished from inbox" >&2
      fi
  else
      printf "An error occured while downloading video at ‘%s’ (youtube-dl: %d)\n" ${url} ${ytResult} >&2
      # notmuch tag +failed -- ${msgId}
      # exit $?
  fi
) 9<>"${message}"