socialnetworknews

commit 9585e7f5dd135703518ed91be41ed85a27d8b96a
parent f3dff41eb14c6af626870ba5a97fe167443f76b0
Author: MTRNord <mtrnord1@gmail.com>
Date:   Mon,  5 Feb 2018 20:35:32 +0100

Fix Stream Client, add some docs

Signed-off-by: MTRNord <mtrnord1@gmail.com>

Diffstat:
M
build.sbt
 | 
10
++++++++--
M
src/main/resources/application_example.conf
 | 
6
++----
M
src/main/resources/logback.xml
 | 
3
+++
M
src/main/scala/Main.scala
 | 
12
++++++++++--
M
src/main/scala/twitterCrawler/StreamingApi.scala
 | 
27
+++++++++++++++++++++------
M
src/main/scala/twitterSender/Sender.scala
 | 
2
+-

6 files changed, 45 insertions(+), 15 deletions(-)
diff --git a/build.sbt b/build.sbt
@@ -31,4 +31,10 @@ libraryDependencies ++= Seq(
   "com.github.tototoshi" %% "scala-csv" % "1.3.5"
 )
 enablePlugins(JavaAppPackaging)
-//scalacOptions in (Compile,doc) := Seq("-groups", "-implicits")
-\ No newline at end of file
+scalacOptions in (Compile,doc) := Seq("-groups", "-implicits")
+
+scalacOptions ++= Seq(
+  "-encoding", "UTF-8",
+  "-Ywarn-dead-code",
+  "-Ywarn-inaccessible"
+)
+\ No newline at end of file
diff --git a/src/main/resources/application_example.conf b/src/main/resources/application_example.conf
@@ -14,9 +14,6 @@ twitter {
     "MTRNord/freifunknews"
   ]
 }
-db {
-  url = "jdbc:sqlite:./tweetdb.sqlite"
-}
 akka {
 
-}
+}
+\ No newline at end of file
diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml
@@ -8,4 +8,7 @@
     <root level="info">
         <appender-ref ref="STDOUT" />
     </root>
+    <!--<root level="debug">
+        <appender-ref ref="STDOUT" />
+    </root>-->
 </configuration>
 \ No newline at end of file
diff --git a/src/main/scala/Main.scala b/src/main/scala/Main.scala
@@ -16,6 +16,12 @@ import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.duration.DurationInt
 import scala.util.{Failure, Success}
 
+/**
+  * @author MTRNord <freifunknews@nordgedanken.blog>
+  * @todo rework to make this more generic
+  * @version 0.1.0
+  *
+  */
 object Main extends App {
 
   val consumerToken =
@@ -59,7 +65,7 @@ object Main extends App {
         consumerToken = consumerToken
       )
 
-      val sender = new twitterSender.Sender(restClient = restClient)
+      //val sender = new twitterSender.Sender(restClient = restClient)
       //sender.sendHelloWorld
 
       val stream = new twitterCrawler.StreamingApi(
@@ -72,7 +78,9 @@ object Main extends App {
     case Failure(err) => println(err.toString)
   }
 
-  //Keep alive workaround
+  /**
+    * Keep alive workaround
+    */
   val waitFunc = Future {
     while (true) {
       Thread.sleep(1000)
diff --git a/src/main/scala/twitterCrawler/StreamingApi.scala b/src/main/scala/twitterCrawler/StreamingApi.scala
@@ -4,7 +4,7 @@ import java.io.File
 import java.text.SimpleDateFormat
 import java.util.Date
 
-import com.danielasfregola.twitter4s.entities.streaming.common.DisconnectMessage
+import com.danielasfregola.twitter4s.entities.streaming.common.{DisconnectMessage, LimitNotice, WarningMessage}
 import com.danielasfregola.twitter4s.entities.{Tweet, User}
 import com.danielasfregola.twitter4s.http.clients.streaming.TwitterStream
 import com.danielasfregola.twitter4s.{TwitterRestClient, TwitterStreamingClient}
@@ -35,8 +35,9 @@ class StreamingApi(val streamingClient: TwitterStreamingClient,
     * @return
     */
   def fetchTweets: Future[TwitterStream] = {
-    val trackedWords: Seq[String] =
-      conf.getStringList("twitter.trackedWords").asScala
+    val trackedWordsConf =
+      conf.getStringList("twitter.trackedWords")
+    var trackedWords: Seq[String] = Seq()
     val trackedLists: List[String] =
       conf.getStringList("twitter.lists").asScala.toList
     val trackedUsers: Seq[Long] = Await.result(
@@ -44,6 +45,10 @@ class StreamingApi(val streamingClient: TwitterStreamingClient,
       scala.concurrent.duration.Duration.Inf
     )
 
+    trackedWordsConf.forEach((s: String) =>{
+      trackedWords = trackedWords :+ s
+    })
+
     println(
       s"Launching streaming session with tracked keywords: $trackedWords\r\n" +
         s"And with tracked Users: $trackedUsers"
@@ -55,7 +60,13 @@ class StreamingApi(val streamingClient: TwitterStreamingClient,
         this.saveIDforLater(tweet)
         println("Done Saving")
       case disconnect: DisconnectMessage =>
-        println("Disconnect: ", disconnect.disconnect.reason)
+        println("Disconnect: ", disconnect.disconnect)
+      case limit: LimitNotice =>
+        println("Limit: ", limit)
+      case warning: WarningMessage =>
+        println("Warning: ", warning.warning)
+      case default =>
+        println(default)
     }
   }
 
@@ -68,10 +79,14 @@ class StreamingApi(val streamingClient: TwitterStreamingClient,
   private def saveIDforLater(tweet: Tweet): Unit = {
     val currentDate =
       new Date(DateTime.now(DateTimeZone.UTC).getMillis)
-    val today = new SimpleDateFormat("dd/MM/yyyy").format(currentDate)
+    val today = new SimpleDateFormat("dd_MM_yyyy").format(currentDate)
+    val dir = new File("data/")
+    if (!dir.exists()) {
+      dir.mkdir()
+    }
     val todaysTweets = new File(s"data/tweets.$today")
     if (!todaysTweets.exists()) {
-      todaysTweets
+      todaysTweets.createNewFile()
     }
     val writer = CSVWriter.open(todaysTweets, append = true)
     writer.writeRow(List(tweet.id_str, currentDate.toString))
diff --git a/src/main/scala/twitterSender/Sender.scala b/src/main/scala/twitterSender/Sender.scala
@@ -9,7 +9,7 @@ class Sender(val restClient: TwitterRestClient) {
   def sendHelloWorld: Tweet = {
     Await.result(
       restClient.createTweet(
-        status = "Hello world vom Scalar Freifunk News Projekt!"
+        status = "Test123"
       ),
       scala.concurrent.duration.Duration.Inf
     )