summaryrefslogtreecommitdiff
path: root/odin/strm
diff options
context:
space:
mode:
Diffstat (limited to 'odin/strm')
-rwxr-xr-xodin/strm/cleanup_youtube61
-rwxr-xr-xodin/strm/download_youtube337
2 files changed, 398 insertions, 0 deletions
diff --git a/odin/strm/cleanup_youtube b/odin/strm/cleanup_youtube
new file mode 100755
index 00000000..0df15d0b
--- /dev/null
+++ b/odin/strm/cleanup_youtube
@@ -0,0 +1,61 @@
1#!@zsh@/bin/zsh
2
3function notmuch {
4 while true; do
5 result=$($_env NOTMUCH_CONFIG=${HOME}/.notmuch-rss-config @notmuch@/bin/notmuch "$@" 2>&1)
6 if ! [[ $result =~ "already locked" ]]; then
7 echo -nE $result
8 return
9 fi
10 sleep 2
11 done
12}
13
14tagFile=""
15
16function cleanup {
17 [[ -n "${tagFile}" ]] && rm -fv ${tagFile}
18}
19
20dir=/srv/media/youtube
21maxsize=$((1024 * 1024 * 1024 * 50))
22
23while [[ $(du -bs $dir | awk '{ print $1; }') -gt $maxsize ]]; do
24 find $dir -type f -not -path $dir/CACHEDIR.TAG -print0 | \
25 xargs -0 -- $_stat --printf '%W %Y %Z %X %n\0' | \
26 sort -t '\0' -z | \
27 awk -F '\0' '{ gsub("^\\S+\\s+\\S+\\s+\\S+\\s+\\S+\\s+", "", $1); printf "%s\0", $1; }' | \
28 xargs -0 -- rm -v
29done
30
31find $dir -xtype l -delete
32find $dir -type d -empty -delete
33
34notmuch search --output=messages --format=text 'is:cached' | \
35 while read id; do
36 untag=false
37 url=$(notmuch show --format=raw "$id" | grep 'http://odin.asgard.yggdrasil/youtube' | sed -r 's|^.*href="http://odin.asgard.yggdrasil/youtube/([^"]+)".*$|\1|' | sed -f /usr/lib/url_unescape.sed)
38
39 printf "%s\n ‘%s’\n" ${id} ${dir}/${url}
40
41 if [[ -z "${url}" ]]; then
42 printf " Could not extract filename.\n"
43 untag=true
44 fi
45
46 if [[ -n "${url}" && ! -e "${dir}/${url}" ]]; then
47 printf " File vanished\n"
48 untag=true
49 fi
50
51 if ${untag}; then
52 if [[ -z "${tagFile}" ]]; then
53 tagFile=$(mktemp --tmpdir $$.tags.XXXXXXXXXX)
54 printf "Using %s\n" ${tagFile}
55 fi
56
57 { printf "-cached -- %s\n" ${id} >> ${tagFile} } && printf " Tagging ‘-cached’...\n"
58 fi
59 done
60
61[[ -n "${tagFile}" ]] && notmuch tag --batch --input=${tagFile}
diff --git a/odin/strm/download_youtube b/odin/strm/download_youtube
new file mode 100755
index 00000000..33e9a454
--- /dev/null
+++ b/odin/strm/download_youtube
@@ -0,0 +1,337 @@
1#!@zsh@/bin/zsh
2
3alwaysTranscode=false
4
5ffmpeg() { { { @ffmpeg-full@/bin/ffmpeg $@ 1>&3 } 2>&1 | stdbuf -o 0 tr '\r' '\n' | grep -v --line-buffered -E '^$' 1>&2 } 3>&1 }
6pv() { { { @pv@/bin/pv -D 2 -i 2 -w 100 -H 1 -f $@ 1>&3 } 2>&1 | stdbuf -o 0 tr '\r' '\n' | grep -v --line-buffered -E '^$' 1>&2 } 3>&1 }
7trimName() { prefix=$1; shift; printf "%s%s" ${prefix} $(awk -v len=$((19 - $#prefix)) '{ if (length($0) > len) print substr($0, 1, len-3) "..."; else print; }' <<<"${(j: :)@}")}
8
9function notmuch {
10 while true; do
11 result=$(env NOTMUCH_CONFIG=${HOME}/.notmuch-rss-config @notmuch@/bin/notmuch "$@" 2>&1)
12 if ! [[ $result =~ "already locked" ]]; then
13 echo -nE $result
14 return
15 fi
16 sleep 2
17 done
18}
19
20logTag=${0:t}
21
22exec 1> >(logger -t "$logTag" -p news.notice)
23exec 2> >(logger -t "$logTag" -p news.error)
24
25debug() { logger -t "$logTag" -p news.debug }
26warn() { logger -t "$logTag" -p news.warn }
27
28typeset -a cleanupCmds
29cleanupCmds=()
30
31function doCleanup {
32 for cmd (${cleanupCmds}); do
33 # print -- ${cmd} | debug
34 eval $cmd | debug
35 done
36}
37
38function cleanup() {
39 local cmd
40 cmd=""
41 for arg ($@); do
42 [[ -n ${cmd} ]] && cmd="$cmd "
43 cmd="${cmd}${(qq)arg}"
44 done
45
46 cleanupCmds+=(${cmd})
47}
48
49mungefilename ()
50{
51 tr "\`\"' /[:upper:]" "_____[:lower:]" <<<${@} | sed -r 's/[^0-9a-z\!\#\$\%\&\(\)\*\+\,\-\.\:\;\<\=\>\?\@\[\]\~\^\_\{\}\|]//g; s/_+/_/g'
52}
53
54[[ $#@ -le 0 || $#@ -gt 1 ]] && exit 2
55
56msgId=$1
57message=$(notmuch search --output=files --duplicate=1 $msgId)
58[[ ! -e $message ]] && exit 1
59
60printf ">>> %s <<<\n %s\n" "${msgId}" "${message}"
61
62( print " waiting for lock..."
63
64 if ! flock -xn 9; then
65 print " could not acquire lock."
66 exit 0
67 fi
68
69 print " locked."
70
71 trap doCleanup EXIT
72
73 typeset -a msgTags
74 msgTags=($(notmuch search --output=tags $msgId ))
75
76 if [[ ${msgTags[(i)cached]} -le $#msgTags ]]; then
77 print "Message in Cache" | warn
78 exit 0
79 fi
80
81 if [[ 'base64' == $(sed '/^Content-Transfer-Encoding: */!d; s///;q' ${message}) ]]; then
82 tmpFile=$(mktemp --tmpdir=/home/gkleen/rss/tmp .writeOut.XXXXXX)
83 printf "Decoding base64 message content for ‘%s’" ${message} | debug
84 sed '/^Content-Transfer-Encoding: */d; /^$/q' ${message} >! $tmpFile
85 sed '1,/^$/d' ${message} | base64 -d >> $tmpFile
86 mv -v $tmpFile ${message}
87 fi
88
89 from=$(mungefilename $(awk '/^From/ { gsub("^\"", "", $2); print $2; exit; }' "${message}"))
90 if grep -q "<p>Enclosure: <a" "${message}"; then
91 url=$(tr -d '\n' < "${message}" | sed -r 's/^.*<p>Enclosure: <a[^>]+href=[^">]*"([^"]+)".*$/\1/')
92 else
93 url=$(awk '/^X-RSS-URL/ { print $2; exit; }' "${message}")
94 fi
95
96 sld=$(cut -d '/' -f 3 <<<${url} | rev | cut -d '.' -f 1-2 | rev)
97
98 if [[ -e ${HOME}/.dl-backup ]] && grep -q ${sld} ${HOME}/.dl-backup; then
99 printf "Was told to back up off %s\n" ${sld} | warn
100 exit 0
101 else
102 printf "Proceeding on %s\n" ${sld} | debug
103 fi
104
105 dir="/srv/media/youtube"
106 if ! { [[ -d $dir ]] || mkdir -p $dir }; then
107 exit 1
108 fi
109
110 formatString='bestvideo[width<=2560][height<=1440][fps<=60][vcodec=vp9]+bestaudio[acodec=opus]/bestvideo[width<=2560][height<=1440][fps<=60]+bestaudio/best[width<=2560][height<=1440][fps<=60]/best'
111
112 if [[ ${msgTags[(i)tv]} -le $#msgTags ]]; then
113 formatString="bestvideo[width<=2560][height<=1440][fps<=60][vcodec=h264]+bestaudio[acodec=aac]/bestvideo[width<=2560][height<=1440][fps<=60][vcodec=mpeg]+bestaudio[acodec=aac]/mp4[width<=2560][height<=1440][fps<=60][vcodec=h264][acodec=aac]/mp4[width<=2560][height<=1440][fps<=60][vcodec=mpeg][acodec=aac]/${formatString}"
114 fi
115
116 typeset -a args
117 args=(--no-playlist --add-metadata --newline --mark-watched -f ${formatString})
118
119 youtube-dl ${args} --get-filename -o $'%(extractor)s\n%(id)s\n%(format)s\n%(title)s\n%(ext)s' -- ${url} | {
120 oldIFS=${IFS}
121 export IFS=
122
123 read extractor
124 read videoId
125 read format
126 read title
127 read fileExtension
128
129 export IFS=${oldIFS}
130 } || {
131 ret=$?
132 printf "An error occured while determining video metadata (exitcode: %d)\n" ${ret} >&2
133 notmuch tag +failed -- ${msgId}
134 exit $?
135 }
136
137 if [[ "${extractor}" == "generic" ]]; then
138 title=$(grep -E '^Subject' "${message}" | head -n 1 | cut -d ' ' -f 2-)
139 fi
140
141 filename="${dir}/${title}-${extractor}.${videoId}.${from}.${fileExtension}"
142 filename=$(tr $'\n\0\/' ' ' <<<${filename} | tr -d $'!')
143
144 printf "%s\n%s via %s: %s\nExpecting ‘%s’\n" ${title} ${videoId} ${extractor} ${format} ${filename:t}
145
146 if [[ -n ${extractor} && -n ${videoId} ]]; then
147 printf "Searching for %s on %s in cache...\n" ${videoId} ${extractor} | debug
148
149 typeset -a messages
150 messages=($(notmuch search --output=messages -- from:${extractor} and ${videoId} and is:cached))
151
152 if [[ ${#messages} -ne 0 ]]; then
153 printf "Video already cached: %s\n" "${messages[*]}" | warn
154 else
155 printf "Video not yet cached\n" | debug
156 fi
157 fi
158
159 youtube-dl $args -o "${filename:r}.%(ext)s" -- ${url} 2>&1 </dev/null | stdbuf -o0 tr '\r' '\n'
160 ytResult=$?
161
162 if [[ ! -e ${filename} && -n ${filename} ]]; then
163 for f (${filename:r}.*(N)); do
164 shouldDelete=false
165
166 grep -qE '\.f[0-9]+\.' <<<${f} && shouldDelete=true
167 grep -qE '\.part(-Frag[0-9]+)?$' <<<${f} && shouldDelete=true
168 grep -qE '\.temp\.[^\.]+$' <<<${f} && shouldDelete=true
169
170 if ${shouldDelete}; then
171 rm -v ${f}
172 else
173 filename=${f}
174 fi
175 done
176 fi
177
178 [[ -n ${filename} ]] && printf "Found ‘%s’\n" ${filename:t} || debug
179
180 # newFilename=${filename:h}/$(mungefilename ${filename:t})
181 # if [[ "${filename}" != "${newFilename}" ]]; then
182 # mv -v ${filename} ${newFilename} && filename=${newFilename}
183 # fi
184
185 if [[ -n "${filename}" && -e "${filename}" && ${ytResult} -eq 0 ]]; then
186 max_vol=$( \
187 pv -N "$(trimName "vol:" ${title})" ${filename} | ffmpeg -i pipe:0 -af "volumedetect" -vn -f null /dev/null 2>&1 | \
188 grep 'max_volume' | sed -r 's/^.*max_volume: ([-0-9\.]+) dB$/\1/'
189 )
190 printf "Maximum volume: %.2fdB\n" "${max_vol}"
191 [[ -n "${max_vol}" ]] || max_vol=0
192 bare_amp=$(printf "%.2f" $(($max_vol * (-1))))
193 amp=$(printf "volume=%sdB" "${bare_amp}")
194
195 typeset -a extensions
196 extensions=("mkv" "mp3" "mp4" "webm")
197
198 if [[ ${msgTags[(i)tv]} -le $#msgTags ]]; then
199 extensions=("mp4")
200 fi
201
202 if [[ "${bare_amp}" -ne 0 || ${extensions[(i)${filename:e}]} -gt ${#extensions} ]]; then
203 printf "Transcoding ‘%s’" ${title} | debug
204 printf "%d %d %d/%d(%d)…\n" \
205 $([[ $($alwaysTranscode; print $?) -eq 0 ]]; print $?) \
206 $([[ "${bare_amp}" -ne 0 ]]; print $?) \
207 ${extensions[(i)${filename:e}]} ${#extensions} \
208 $([[ ${extensions[(i)${filename:e}]} -gt ${#extensions} ]]; print $?) \
209 | debug
210 tempfile=$(mktemp --tmpdir=${filename:h} .transcode.${filename:t:r}.$$.XXXXXX.${filename:e})
211 cleanup rm -v -- "${tempfile}"
212 mv -vf "${filename}" "${tempfile}" | debug
213
214 typeset -A fileInfo
215 for line ($(ffprobe -v error -show_format -show_streams -show_entries stream=codec_name,codec_type:format=:stream_tags=:stream_disposition=:format_tags= -of flat=h=0 -- ${tempfile})); do
216 fileInfo[${line%=*}]=${(Q)line#*=}
217 done
218
219 ext=${extensions[1]}
220 filename=${filename%.*}.${ext}
221 typeset -a args
222 args=(-y)
223 typeset -a prePass
224 prePass=()
225 typeset -a cargs
226 cargs=()
227 typeset -a p1args
228 p1args=()
229 typeset -a p2args
230 p2args=()
231 for formatKey (${(k)fileInfo}); do
232 [[ ${formatKey} =~ "stream.([0-9]+).codec_type" ]] || continue
233 local i=$match[1]
234 local cType=${fileInfo[${formatKey}]}
235 local cName=${fileInfo[stream.${i}.codec_name]}
236
237 printf "Stream %d: %s (%s)\n" ${i} ${cName} ${cType} | debug
238
239 if [[ ${cType} == "video" ]]; then
240 if [[ ${ext} == "mkv" ]]; then
241 case ${cName} in
242 h264|vp9|png)
243 cargs+=(-c:${i} copy)
244 ;;
245 *)
246 # p1args=(-pass 1 -threads 8 -speed 4 -tile-columns 6 -frame-parallel 1)
247 # prePass+=(-c:${i} libvpx-vp9 -b:${i} 0 -crf:${i} 33)
248 # p2args+=(-pass 2 -threads 8 -speed 2 -tile-columns 6 -frame-parallel 1 -auto-alt-ref 1 -lag-in-frames 25)
249 # cargs+=(-c:${i} libvpx-vp9 -b:${i} 0 -crf:${i} 33)
250 cargs+=(-c:${i} libvpx-vp9 -b:${i} 2M -threads 8 -tile-columns 6 -frame-parallel 1)
251 ;;
252 esac
253 elif [[ ${ext} == "mp4" ]]; then
254 case ${cName} in
255 mpeg|h264)
256 cargs+=(-c:${i} copy)
257 ;;
258 *)
259 p2args+=(-strict -2)
260 cargs+=(-c:${i} libx264)
261 ;;
262 esac
263 fi
264 elif [[ ${cType} == "audio" ]]; then
265 if [[ ${ext} == "mkv" ]]; then
266 if [[ ( ${cName} == opus || ${cName} == flac || ${cName} == vorbis ) && "${bare_amp}" -eq 0 ]]; then
267 cargs+=(-c:${i} copy)
268 else
269 p2args+=(-vbr on -compression_level 10)
270 cargs+=(-c:${i} libopus -b:${i} 256K)
271 fi
272 elif [[ ${ext} == "mp4" ]]; then
273 if [[ ${cName} == aac && "${bare_amp}" -eq 0 ]]; then
274 cargs+=(-c:${i} copy)
275 else
276 cargs+=(-c:${i} aac)
277 fi
278 fi
279 fi
280 done
281
282 [[ "${bare_amp}" -ne 0 ]] && p2args+=(-af "${amp}")
283
284 if [[ $#prePass -gt 0 ]]; then
285 args+=(-v info -i ${tempfile})
286
287 oldPwd=${PWD}
288 cd $(mktemp -d --tmpdir "transcode.${0:t}.$$.XXXXXX")
289 cleanup rm -rfv -- ${PWD}
290
291 p1args+=(-an -f matroska)
292 print -- ${prePass} ${p1args} | debug
293 ffmpeg ${args} ${prePass} ${p1args} -- /dev/null
294
295 print -- ${cargs} ${p2args} | debug
296 ffmpeg ${args} ${cargs} ${p2args} -- "${filename}"
297
298 cd ${oldPwd}
299 else
300 args+=(-v warning -i pipe:0)
301
302 print -- ${cargs} ${p2args} | debug
303 pv -N "$(trimName "trans:" ${title})" ${tempfile} | ffmpeg ${args} ${cargs} ${p2args} -- "${filename}"
304 fi
305 fi
306
307 chmod -v 644 "${filename}" | debug
308 tmpFile=$(mktemp --tmpdir=/home/gkleen/rss/tmp .insertUrl.$$.XXXXXX)
309 relUrl=$(realpath --relative-to=/srv/media ${filename})
310 typeset -a relUrlComponents
311 relUrlComponents=(${(s./.)relUrl})
312 relUrl=""
313 for urlPiece (${relUrlComponents}); do
314 [[ -n "${relUrl}" ]] && relUrl+="/"
315 relUrl+=$(sed -f ${HOME}/url_escape.sed <<<${urlPiece})
316 done
317 awk -v "link=http://odin.asgard.yggdrasil/${relUrl}" '{ if (r == 0) { r = gsub("href=\"[^\"]+\"", "href=\"" link "\""); }; print; }' "${message}" >! $tmpFile
318 mv -v $tmpFile $message | debug
319 notmuch tag '+cached' -- $msgId && printf "Tagged ‘%s’ as 'cached'" ${msgId}
320
321 if [[ -n "$(notmuch search "tag:inbox AND $msgId")" ]]; then
322 if [[ ${msgTags[(i)tv]} -le $#msgTags ]]; then
323 printf "%s\n%s\n" "Media available on odin" "${title}" \
324 | uux -p -n 'hel!notify-gkleen' -a download_youtube -u normal \
325 && print "Sent notification to hel via uucp"
326 else
327 queue.hel "${filename}" && notmuch tag '-inbox' '-unread' -- $msgId
328 fi
329 else
330 print "Message vanished from inbox" | warn
331 fi
332 else
333 printf "An error occured while downloading video at ‘%s’ (exitcode: %d)\n" ${url} ${ytResult} >&2
334 # notmuch tag +failed -- ${msgId}
335 # exit $?
336 fi
337) 9<>"${message}"