commit 3bc75731516a3a09eae656ff868e8cd554a88875
parent dbf5a340ea9bd615a240cf3e0dac29fc590555e5
Author: Christian Bednarek <eazy@eazy-living.de>
Date: Thu, 26 Jan 2023 17:58:42 +0100
Add files via upload
Diffstat:
| A | setup.sh | | | 130 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
| A | test.wav | | | 0 | |
| A | transcribe.sh | | | 164 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
3 files changed, 294 insertions(+), 0 deletions(-)
diff --git a/setup.sh b/setup.sh
@@ -0,0 +1,130 @@
+#!/bin/bash
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+cd $SCRIPT_DIR
+
+if [ -f ./fyyd.cfg ]
+ then
+ source "./fyyd.cfg"
+fi
+
+
+export LC_NUMERIC="en_US.UTF-8"
+
+# Check ffmpeg etc
+
+if ! command -v ffmpeg &> /dev/null
+then
+ echo "Can't find ffmpeg. Please install."
+ exit
+fi
+
+if ! command -v git &> /dev/null
+then
+ echo "Can't find git. Please install."
+ exit
+fi
+
+if ! command -v curl &> /dev/null
+then
+ echo "Can't find curl. Please install."
+ exit
+fi
+
+if [ ! -f "whisper.cpp/whisper.cpp" ]
+ then
+ echo "downloading whisper.cpp"
+ git clone https://github.com/ggerganov/whisper.cpp
+
+ echo "compiling whisper..."
+ cd whisper.cpp
+ make
+
+ echo "downloading the model"
+ ./models/download-ggml-model.sh medium
+ else
+ cd whisper.cpp
+fi
+
+THREADS=0
+THREADS=`getconf _NPROCESSORS_ONLN`
+if [ $? -ne 0 ]
+ then
+ THREADS=`getconf NPROCESSORS_ONLN`
+fi
+
+
+if [ $THREADS -eq 0 ]
+ then
+ echo -n "could not find number of threads. how many to use for transcription? (number or return to let whisper utilize cpu as guessed): "
+ else
+ echo -n "maximum of $THREADS threads found. how many to use for transcription? (number of threads or return for $THREADS) : "
+fi
+
+read inputthreads
+
+if [ ! -z "$inputthreads" ]
+ then
+ echo "leer"
+ THREADS=$inputthreads
+fi
+
+THREAD_OPT=""
+
+if [ "$THREADS" -ne 0 ]
+ then
+ THREAD_OPT=" -t $THREADS"
+fi
+
+if [ -z "$ACCESSTOKEN" ]
+ then
+ echo -n "please provide the accesstoken for fyyd (return to set later in fyyd.cfg): "
+ read token_input
+ if [ ! -z "$token_input" ]
+ then
+ echo "READ"
+ ACCESSTOKEN=$token_input
+ fi
+fi
+
+START=`date +%s`
+
+echo -e "\nstarting test. this might take some minutes, please wait...\n"
+nice -n 18 ./main -m models/ggml-medium.bin -su $THREAD_OPT -l de ../test.wav 2>/dev/null
+
+if [ $? -ne 0 ]
+ then
+ echo "error transcribing. stopping"
+
+fi
+
+
+DURATION=300
+END=`date +%s`
+TOOK=$(($END-$START))
+RATE=`echo "$DURATION/$TOOK" | bc -l`
+
+echo -e "------------------------------------------------------------------------------------------------------\n"
+
+echo -n "transcription took $TOOK seconds for 300 seconds audio. that is "
+printf "%.2f" $(echo "$DURATION/$TOOK" | bc -l)
+echo " times faster than realtime audio."
+
+if (( $(echo "$RATE < 1.5" |bc -l) ))
+ then
+ echo "that's very slow and you should maybe NOT transcribe with this computer."
+ elif (( $(echo "$RATE < 2" |bc -l) ))
+ then
+ echo "that's ok, but you should raise the number of threads if possible."
+ else
+ echo "that's great. let's transcribe some podcasts!"
+fi
+
+echo
+cd ..
+
+rm -f fyyd.cfg
+echo "THREADS=$THREADS" >> ./fyyd.cfg
+echo "ATOKEN=$ACCESSTOKEN" >> ./fyyd.cfg
+echo "PIDFILE=~/.fyyd-transcribe.pid" >> ./fyyd.cfg
+echo "MODEL=medium" >> ./fyyd.cfg
diff --git a/test.wav b/test.wav
Binary files differ.
diff --git a/transcribe.sh b/transcribe.sh
@@ -0,0 +1,164 @@
+#!/bin/bash
+
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+if [ -f ./fyyd.cfg ]
+ then
+ source "./fyyd.cfg"
+ else
+ echo "no config found. please run setup.sh first."
+ exit 0
+fi
+
+
+export LC_NUMERIC="en_US.UTF-8"
+
+trap ctrl_c INT
+trap stop EXIT
+
+function stop {
+ rm $PIDFILE
+}
+
+function ctrl_c() {
+
+ echo "------------------------------"
+ echo "STOPPING... notify fyyd"
+ echo "bye bye"
+ echo "------------------------------"
+ curl -H "Authorization: Bearer $ATOKEN" "https://api.fyyd.de/0.2/transcribe/error/$ID" -d "error=0"
+ rm -f $PIDFILE
+ exit
+}
+
+pid() {
+ if [ -f $PIDFILE ]
+ then
+ PID=$(cat $PIDFILE)
+ ps -p $PID > /dev/null 2>&1
+ if [ $? -eq 0 ]
+ then
+ echo "Process already running"
+ exit 1
+ else
+ ## Process not found assume not running
+ echo $$ > $PIDFILE
+ if [ $? -ne 0 ]
+ then
+ echo "Could not create PID file"
+ exit 1
+ fi
+ fi
+ else
+ echo $$ > $PIDFILE
+ if [ $? -ne 0 ]
+ then
+ echo "Could not create PID file"
+ exit 1
+ fi
+ fi
+}
+
+pid
+
+echo "Starting engines! Let's transcribe some episodes"
+cd whisper.cpp
+
+while :
+do
+
+ #------------------------------------------------------------------------------------
+ # get data for one episode to transcribe from fyyd.de
+ #------------------------------------------------------------------------------------
+
+ echo "getting data from fyyd"
+ read ID URL TOKEN LANG DURATION TITLE < <(echo $(curl -s -H "Authorization: Bearer $ATOKEN" "https://api.fyyd.de/0.2/transcribe/next" | jq -r '.data | .[]'))
+
+ # exit if nothing to do
+
+ if [ -z $ID ]
+ then
+ echo "nothing to transcribe. exit!"
+ exit 0;
+ fi
+
+ #------------------------------------------------------------------------------------
+ # download episode
+ #------------------------------------------------------------------------------------
+
+ echo "starting download of episode $ID, \"$TITLE\", duration $DURATION seconds"
+
+ curl -s -L $URL > $TOKEN
+ if [ $? -ne 0 ]
+ then
+ echo "error downloading"
+ curl -H "Authorization: Bearer $ATOKEN" "https://api.fyyd.de/0.2/transcribe/error/$ID" -d "error=900"
+ continue
+
+ fi
+
+
+ #------------------------------------------------------------------------------------
+ # convert whatever was donwloaded to 16kHz WAV
+ #------------------------------------------------------------------------------------
+
+ echo "converting to wav"
+
+ ffmpeg -y -i $TOKEN -acodec pcm_s16le -ac 1 -ar 16000 $TOKEN.wav >/dev/null 2>/dev/null
+
+ if [ $? -eq 1 ]
+ then
+ echo "error converting to wav"
+ curl -H "Authorization: Bearer $ATOKEN" "https://api.fyyd.de/0.2/transcribe/error/$ID" -d "error=901"
+ continue
+ fi
+
+ rm $TOKEN
+
+ #------------------------------------------------------------------------------------
+ # transcribe wav to vtt
+ #------------------------------------------------------------------------------------
+
+ START=`date +%s`
+
+ echo "starting whisper"
+ nice -n 18 ./main -su -m models/ggml-$MODEL.bin -t $THREADS -l $LANG -ovtt $TOKEN.wav >/dev/null 2>/dev/null
+
+ if [ $? -ne 0 ]
+ then
+ echo "error transcribing"
+ curl -H "Authorization: Bearer $ATOKEN" "https://api.fyyd.de/0.2/transcribe/error/$ID" -d "error=902"
+ continue
+
+ fi
+
+ END=`date +%s`
+ TOOK=$(($END-$START))
+
+ echo -n "Rate: "
+ printf "%.2f" $(echo "$DURATION/$TOOK" | bc -l)
+ echo "x"
+
+
+ #------------------------------------------------------------------------------------
+ # push transcript to fyyd
+ #------------------------------------------------------------------------------------
+
+ curl -H "Authorization: Bearer $ATOKEN" "https://api.fyyd.de/0.2/transcribe/set/$ID" --data-binary @$TOKEN.wav.vtt
+
+ rm $TOKEN.wav
+ rm $TOKEN.wav.vtt
+
+ if [ -f ~/.fyyd-stop ]; then
+ rm ~/.fyyd-stop
+ echo "stopping hard"
+ exit
+ fi
+
+ echo "--------------------------------------------------------------"
+ sleep 2
+
+done
+
+rm $PIDFILE
