transcribe.sh (4268B)
1 #!/bin/bash 2 3 if [ -f ./trancribe.cfg ] 4 then 5 source "./trancribe.cfg" 6 else 7 echo "no config found. please run setup.sh first." 8 exit 0 9 fi 10 11 12 export LC_NUMERIC="en_US.UTF-8" 13 14 trap ctrl_c INT 15 16 function ctrl_c() { 17 echo "------------------------------" 18 echo "STOPPING..." 19 echo "bye bye" 20 echo "------------------------------" 21 exit 22 } 23 24 echo "Starting engines! Let's transcribe some episodes" 25 pushd whisper.cpp || exit 26 27 #------------------------------------------------------------------------------------ 28 # get data for one episode to transcribe from matrix youtube 29 #------------------------------------------------------------------------------------ 30 31 echo "getting data from youtube" 32 mkdir -p ./playlist 33 touch ./playlist/downloaded.txt 34 mkdir -p ./playlist_normalized 35 pushd ./playlist || exit 36 37 #PLAYLIST_URL="https://www.youtube.com/playlist?list=PLl5dnxRMP1hXBHqokHol6DTVIbnsf57Mr" 38 #PLAYLIST_URL="https://www.youtube.com/watch?v=HlsMMzTFZ_A" 39 PLAYLIST_URL="https://www.youtube.com/@Matrixdotorg" 40 41 python3 -m yt_dlp "${PLAYLIST_URL}" -x -f ba --audio-format wav --audio-quality 0 -o "%(id)s.%(ext)s" --concurrent-fragments 3 --download-archive ./downloaded.txt --live-from-start --extractor-args youtube:player_client=android 42 popd || exit 43 44 # exit if nothing to do 45 46 if [ -z "$(find "./playlist/" -maxdepth 1 -type f 2>/dev/null)" ]; 47 then 48 echo "nothing to transcribe. exit!" 49 exit 0; 50 fi 51 52 files_array=(./playlist/*.wav) 53 for next_file in "${files_array[@]}" 54 do 55 FILENAME=$(basename "${next_file}" ".wav") 56 57 # Skip file if output exists 58 if [ -f "./output/${FILENAME}.vtt" ] || [ -f "./output/${FILENAME}.txt" ]; then 59 rm "${next_file}" 60 fi 61 done 62 63 echo "normalize" 64 files_already_done=($(find . -wholename "./playlist_normalized/*.wav" -type f | tr '\n' ' ' | sed -e 's/playlist_normalized/playlist/g')) 65 files=$(find . -wholename "./playlist/*.wav" -type f | tr '\n' ' ') 66 for already_done_file in "${files_already_done[@]}" 67 do 68 files=$(echo "${files}" | sed -e "s@${already_done_file}@@g") 69 done 70 files_out=$(echo "${files}" | sed -e 's/playlist/playlist_normalized/g') 71 72 [[ $files = *[!\ ]* ]] && python3 -m ffmpeg_normalize ${files} -p -o ${files_out} -ar 16000 73 74 ## This logic is used to not process stuff twice 75 #files_already_done=($(find . -wholename "./output/*.vtt" -type f | tr '\n' ' ' | sed -e 's/output/playlist_normalized/g' | sed -e 's/vtt/wav/g')) 76 files_already_done=($(find . -wholename "./output/*.txt" -type f | tr '\n' ' ' | sed -e 's/output/playlist_normalized/g'| sed -e 's/txt/wav/g')) 77 files=$(find . -wholename "./playlist_normalized/*.wav" -type f | tr '\n' ' ') 78 for already_done_file in "${files_already_done[@]}" 79 do 80 files=$(echo "${files}" | sed -e "s@${already_done_file}@@g") 81 done 82 out_files=($(echo "${files}" | sed -e 's/playlist_normalized/output/g' | sed -e 's/.wav//g')) 83 84 echo "starting whisper" 85 86 if [[ $files = *[!\ ]* ]]; then 87 array_files=(in cafiles) 88 for i in "${!array_files[@]}"; do 89 echo "Starting with ${array_files[$1]}" 90 #if ! nice -n 18 ./main -m "models/ggml-${MODEL}.bin" -t "$THREADS" -l en -ovtt -pc "${files}.wav"; #>/dev/null 2>/dev/null; 91 if ! nice -n 18 ./main -m "models/ggml-${MODEL}.bin" -t "$THREADS" -l en -otxt -ovtt -pc --file "${array_files[$i]}" --output-file "${out_files[$i]}" -et 3.0; #>/dev/null 2>/dev/null; 92 then 93 echo "error transcribing" 94 fi 95 96 echo "cleanup" 97 files_array=(./playlist_normalized/*.wav) 98 for next_file in "${files_array[@]}" 99 do 100 FILENAME=$(basename "${next_file}" ".wav") 101 # Removes some junk from hallucination 102 sed -i -e 's/ Subtitles by the Amara.org community//g' "./output/${FILENAME}.txt" 103 sed -i -e 's/Subtitles by the Amara.org community//g' "./output/${FILENAME}.vtt" 104 105 mkdir ./playlist_bak 106 mkdir ./playlist_normalized_bak 107 mv "./playlist/${FILENAME}.wav" "./playlist_bak/${FILENAME}.wav" 108 mv "./playlist_normalized/${FILENAME}.wav" "./playlist_normalized_bak/${FILENAME}.wav" 109 #rm "./${FILENAME}.vtt" 110 mv "./${FILENAME}.wav.vtt" "./output/${FILENAME}.vtt" 111 mv "./${FILENAME}.wav.txt" "./output/${FILENAME}.txt" 112 done 113 done 114 fi 115 116 popd || exit