socialnetworknews

commit 1f6b7bcf801f835c3f1da93f86718c743c0f6061
parent c3307c7192b5cb0112b0378815c01c28ea85112e
Author: MTRNord <mtrnord1@gmail.com>
Date:   Mon,  5 Feb 2018 18:18:36 +0100

Change Database to simple csv

Signed-off-by: MTRNord <mtrnord1@gmail.com>

Diffstat:
M
.gitignore
 | 
7
+++----
M
build.sbt
 | 
7
++++---
M
src/main/scala/twitterCrawler/StreamingApi.scala
 | 
61
+++++++++++++++++++++++++++++++++++++++++++------------------

3 files changed, 50 insertions(+), 25 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -4,7 +4,6 @@
 # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
 
 # User-specific stuff:
-.idea/**/workspace.xml
 .idea/**/tasks.xml
 .idea/dictionaries
 
@@ -19,7 +18,6 @@
 
 # Gradle:
 .idea/**/gradle.xml
-.idea/**/libraries
 
 # CMake
 cmake-build-debug/
@@ -67,4 +65,5 @@ project/plugins/project/
 *.class
 *.log
 src/main/resources/application.conf
-.idea/
-\ No newline at end of file
+.idea/
+data/
+\ No newline at end of file
diff --git a/build.sbt b/build.sbt
@@ -27,7 +27,8 @@ libraryDependencies ++= Seq(
   "com.typesafe" % "config" % "1.3.2",
   "ch.qos.logback" % "logback-classic" % "1.1.9",
   "com.typesafe.slick" %% "slick" % "3.2.1",
-  "com.typesafe.slick" %% "slick-hikaricp" % "3.2.1"
+  "com.typesafe.slick" %% "slick-hikaricp" % "3.2.1",
+  "com.github.tototoshi" %% "scala-csv" % "1.3.5"
 )
 enablePlugins(JavaAppPackaging)
-scalacOptions in (Compile,doc) := Seq("-groups", "-implicits")
-\ No newline at end of file
+//scalacOptions in (Compile,doc) := Seq("-groups", "-implicits")
+\ No newline at end of file
diff --git a/src/main/scala/twitterCrawler/StreamingApi.scala b/src/main/scala/twitterCrawler/StreamingApi.scala
@@ -1,22 +1,35 @@
 package twitterCrawler
 
+import java.io.File
+import java.text.SimpleDateFormat
+import java.util.Date
+
 import com.danielasfregola.twitter4s.entities.streaming.common.DisconnectMessage
 import com.danielasfregola.twitter4s.entities.{Tweet, User}
 import com.danielasfregola.twitter4s.http.clients.streaming.TwitterStream
 import com.danielasfregola.twitter4s.{TwitterRestClient, TwitterStreamingClient}
+import com.github.tototoshi.csv.CSVWriter
 import com.typesafe.config.{Config, ConfigFactory}
 import org.joda.time.{DateTime, DateTimeZone}
 
 import scala.collection.JavaConverters._
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.{Await, Future}
-import slick.jdbc.SQLiteProfile.api._
-
-import scala.concurrent.duration.Duration
 
+/**
+  * StreamingAPI class is the connector for the Twitter Streaming Api
+  *
+  * It mainly has the purpose to get Tweets of the users from the Lists and Hashtags defined inside the Config
+  */
 class StreamingApi(val streamingClient: TwitterStreamingClient,
                    val restClient: TwitterRestClient) {
   val conf: Config = ConfigFactory.load()
+
+  /**
+    * fetchTweets is a async function, that listens on Twitter's Streaming API for the defined Lists and Hastags
+    *
+    * @return
+    */
   def fetchTweets: Future[TwitterStream] = {
     val trackedWords: Seq[String] =
       conf.getStringList("twitter.trackedWords").asScala
@@ -35,29 +48,41 @@ class StreamingApi(val streamingClient: TwitterStreamingClient,
     streamingClient.filterStatuses(tracks = trackedWords, follow = trackedUsers) {
       case tweet: Tweet =>
         println(tweet.text)
-        //this.saveToDatabase(tweet)
+        this.saveIDforLater(tweet)
         println("Done Saving")
       case disconnect: DisconnectMessage =>
         println("Disconnect: ", disconnect.disconnect.reason)
     }
   }
 
-  private def saveToDatabase(tweet: Tweet): Unit = {
-    val db = Database.forConfig("db")
-    try {
-      val tweets = TableQuery[database.Tweets]
-      val currentDate =
-        new java.sql.Date(DateTime.now(DateTimeZone.UTC).getMillis)
-      val sqlSetup = DBIO.seq(
-        // Create the tables, including primary and foreign keys
-        tweets.schema.create,
-        tweets += (tweet.id.toInt, "", "", "", "", currentDate)
-      )
-      val setupFuture = db.run(sqlSetup)
-      Await.result(setupFuture, Duration.Inf)
-    } finally db.close()
+  /**
+    * saveIDforLater saves the tweetID and indexing Date to a csv as soon as the Tweet gets published
+    *
+    * @todo implement
+    * @param tweet holds the currently processed Tweet
+    */
+  private def saveIDforLater(tweet: Tweet): Unit = {
+    val currentDate =
+      new Date(DateTime.now(DateTimeZone.UTC).getMillis)
+    val today = new SimpleDateFormat("dd/MM/yyyy").format(currentDate)
+    val todaysTweets = new File(s"data/tweets.$today")
+    if (!todaysTweets.exists()) {
+      todaysTweets
+    }
+    val writer = CSVWriter.open(todaysTweets, append = true)
+    writer.writeRow(List(tweet.id_str, currentDate.toString))
+
+    writer.close()
   }
 
+  /**
+    * getListUsers is used to ask the Twitter API about what users are in a List
+    *
+    * Uses the Twitter Rest API
+    *
+    * @param trackedLists holds the list slugs and corresponding Users for all tracked lists
+    * @return
+    */
   private def getListUsers(trackedLists: List[String]): Future[Seq[Long]] = {
     Future {
       var trackedUsers: Seq[Long] = Seq()