socialnetworknews

commit 50f5d07780c259037edf915abf727986b29837c7
parent 00fc7b641de603767f7b95cda62727ab40865c0b
Author: MTRNord <mtrnord1@gmail.com>
Date:   Tue,  6 Feb 2018 19:19:48 +0100

Log to file and some visibility improvements

Signed-off-by: MTRNord <mtrnord1@gmail.com>

Diffstat:
M
.gitignore
 | 
5
+++--
M
build.sbt
 | 
1
+
M
src/main/resources/logback.xml
 | 
41
++++++++++++++++++++++++++++++++++++-----
M
src/main/scala/Main.scala
 | 
7
+++----
M
src/main/scala/generator/GenerateNewsPaper.scala
 | 
5
++++-
M
src/main/scala/twitterCrawler/StreamingApi.scala
 | 
3
+++

6 files changed, 50 insertions(+), 12 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -66,4 +66,5 @@ project/plugins/project/
 *.log
 src/main/resources/application.conf
 .idea/
-data/
-\ No newline at end of file
+data/
+log/
+\ No newline at end of file
diff --git a/build.sbt b/build.sbt
@@ -33,6 +33,7 @@ libraryDependencies ++= Seq(
   "com.github.tototoshi" %% "scala-csv" % "1.3.5",
   "org.quartz-scheduler" % "quartz" % "2.3.0",
   "org.quartz-scheduler" % "quartz-jobs" % "2.3.0",
+  "org.codehaus.janino" % "janino" % "3.0.7",
 )
 enablePlugins(JavaAppPackaging)
 scalacOptions in (Compile,doc) := Seq("-groups", "-implicits")
diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml
@@ -1,15 +1,46 @@
 
-<configuration>
+<configuration scan="true">
     <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <filter class="ch.qos.logback.core.filter.EvaluatorFilter">
+            <evaluator class="ch.qos.logback.classic.boolex.JaninoEventEvaluator">
+                <expression>return (level == INFO || level == WARN || level == ERROR);</expression>
+            </evaluator>
+            <OnMismatch>DENY</OnMismatch>
+            <OnMatch>ACCEPT</OnMatch>
+        </filter>
         <encoder>
-            <pattern>%d{HH:mm:ss.SSS} SNN %-5level %logger{36} - %msg%n</pattern>
+            <pattern>%d{HH:mm:ss} [%level] [%file:%line] - %msg%n</pattern>
         </encoder>
     </appender>
 
-    <root level="info">
+    <appender name="rollingFile" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <filter class="ch.qos.logback.core.filter.EvaluatorFilter">
+            <evaluator class="ch.qos.logback.classic.boolex.JaninoEventEvaluator">
+                <expression>return (level == INFO || level == WARN || level == ERROR);</expression>
+            </evaluator>
+            <OnMismatch>DENY</OnMismatch>
+            <OnMatch>ACCEPT</OnMatch>
+        </filter>
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <fileNamePattern>log/info-%d{yyyy-MM-dd}.log</fileNamePattern>
+            <maxHistory>30</maxHistory>
+            <totalSizeCap>3GB</totalSizeCap>
+        </rollingPolicy>
+        <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+            <maxFileSize>3MB</maxFileSize>
+        </triggeringPolicy>
+        <encoder>
+            <pattern>%d{HH:mm:ss} [%level] [%file:%line] - %msg%n</pattern>
+        </encoder>
+    </appender>
+
+    <root>
         <appender-ref ref="STDOUT" />
+        <appender-ref ref="rollingFile" />
     </root>
-    <!--<root level="debug">
+    <!--
+    <root level="debug">
         <appender-ref ref="STDOUT" />
-    </root>-->
+    </root>
+    -->
 </configuration>
 \ No newline at end of file
diff --git a/src/main/scala/Main.scala b/src/main/scala/Main.scala
@@ -4,12 +4,11 @@ import com.danielasfregola.twitter4s.entities.{AccessToken, ConsumerToken}
 import com.danielasfregola.twitter4s.{TwitterAuthenticationClient, TwitterRestClient, TwitterStreamingClient}
 import com.typesafe.config.{Config, ConfigFactory}
 import com.typesafe.scalalogging.StrictLogging
-import generator.GenerateNewsPaper
 import twitterCrawler.RestAPISingleton
 
-import scala.concurrent.{Await, Future}
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.duration.DurationInt
+import scala.concurrent.{Await, Future}
 import scala.util.{Failure, Success}
 
 /**
@@ -69,8 +68,8 @@ object Main extends App with StrictLogging {
       )
       stream.fetchTweets
 
-      val test = new GenerateNewsPaper
-      test.execute(null)
+      /*val test = new GenerateNewsPaper
+      test.execute(null)*/
 
       webServer.WebServer.main()
 
diff --git a/src/main/scala/generator/GenerateNewsPaper.scala b/src/main/scala/generator/GenerateNewsPaper.scala
@@ -14,11 +14,12 @@ import scala.concurrent.Await
 class GenerateNewsPaper extends Job with StrictLogging {
   def execute(context: JobExecutionContext): Unit = {
     //TODO parse CSV from yesterday and generate Feed
+    println("=============")
     val reader = this.getReader
     val api = RestAPISingleton.getRestAPI
     reader.foreach(line => {
       val tweetID: Long = line.head.toLong
-      logger.info(tweetID.toString)
+      println("--------------")
       //Get Tweet from id
       val tweets = Await.result(api.getTweet(id = tweetID), scala.concurrent.duration.Duration.Inf)
       val tweet = tweets.data
@@ -36,7 +37,9 @@ class GenerateNewsPaper extends Job with StrictLogging {
       logger.info(s"RetweetCount: $retweet_count")
       logger.info(s"RetweetStatus: $retweeted")
       logger.info(s"FavoriteCount: $favorite_count")
+      println("--------------")
     })
+    println("=============")
   }
 
   private def getReader: CSVReader = {
diff --git a/src/main/scala/twitterCrawler/StreamingApi.scala b/src/main/scala/twitterCrawler/StreamingApi.scala
@@ -69,9 +69,12 @@ class StreamingApi(val streamingClient: TwitterStreamingClient) extends StrictLo
 
     streamingClient.filterStatuses(tracks = trackedWords, follow = trackedUsers) {
       case tweet: Tweet =>
+        println("=============")
+        logger.info("Found a new Tweet... Saving...")
         logger.debug(tweet.text)
         this.saveIDforLater(tweet)
         logger.debug("Done Saving")
+        println("=============")
       case disconnect: DisconnectMessage =>
         logger.warn("Disconnect: ", disconnect.disconnect)
       case limit: LimitNotice =>