commit 3d571873f523a5a482552257079a4038a1a1c0f4
parent 8424a1962b1dc480aee922838678f0d32d45fc7c
Author: MTRNord <mtrnord1@gmail.com>
Date: Tue, 6 Feb 2018 18:09:22 +0100
Prepare Generator and fix some things and update twitter Lib
Signed-off-by: MTRNord <mtrnord1@gmail.com>
Diffstat:
6 files changed, 98 insertions(+), 12 deletions(-)
diff --git a/build.sbt b/build.sbt
@@ -22,9 +22,10 @@ mainClass in (Compile, packageBin) := Some("Main")
mainClass in (Compile, run) := Some("Main")
resolvers += Resolver.sonatypeRepo("snapshots")
+resolvers += Resolver.sonatypeRepo("stable")
resolvers += DefaultMavenRepository
libraryDependencies ++= Seq(
- "com.danielasfregola" %% "twitter4s" % "5.5-SNAPSHOT",
+ "com.danielasfregola" %% "twitter4s" % "6.0-SNAPSHOT",
"com.typesafe" % "config" % "1.3.2",
"ch.qos.logback" % "logback-classic" % "1.1.9",
"com.typesafe.slick" %% "slick" % "3.2.1",
diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml
@@ -1,3 +1,4 @@
+
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
diff --git a/src/main/scala/Main.scala b/src/main/scala/Main.scala
@@ -1,9 +1,11 @@
import com.danielasfregola.twitter4s.entities.enums.AccessType
import com.danielasfregola.twitter4s.entities.enums.AccessType.AccessType
import com.danielasfregola.twitter4s.entities.{AccessToken, ConsumerToken}
-import com.danielasfregola.twitter4s.util.Configurations.{consumerTokenKey, consumerTokenSecret}
import com.danielasfregola.twitter4s.{TwitterAuthenticationClient, TwitterRestClient, TwitterStreamingClient}
+import com.typesafe.config.{Config, ConfigFactory}
import com.typesafe.scalalogging.StrictLogging
+import generator.GenerateNewsPaper
+import twitterCrawler.RestAPISingleton
import scala.concurrent.{Await, Future}
import scala.concurrent.ExecutionContext.Implicits.global
@@ -17,9 +19,9 @@ import scala.util.{Failure, Success}
*
*/
object Main extends App with StrictLogging {
-
+ val conf: Config = ConfigFactory.load
val consumerToken =
- ConsumerToken(key = consumerTokenKey, secret = consumerTokenSecret)
+ ConsumerToken(key = conf.getString("twitter.consumer.key"), secret = conf.getString("twitter.consumer.secret"))
val TwitterAuthClient = new TwitterAuthenticationClient(consumerToken)
val write_access: AccessType = AccessType.Write
@@ -42,8 +44,6 @@ object Main extends App with StrictLogging {
10 seconds
)
- val consumerToken =
- ConsumerToken(key = consumerTokenKey, secret = consumerTokenSecret)
val accessToken = AccessToken(
key = AccessTokenResp.accessToken.key,
secret = AccessTokenResp.accessToken.secret
@@ -62,12 +62,16 @@ object Main extends App with StrictLogging {
//val sender = new twitterSender.Sender(restClient = restClient)
//sender.sendHelloWorld
+ RestAPISingleton.setRestAPI(restClient)
+
val stream = new twitterCrawler.StreamingApi(
- streamingClient = streamingClient,
- restClient = restClient
+ streamingClient = streamingClient
)
stream.fetchTweets
+ val test = new GenerateNewsPaper
+ test.execute(null)
+
webServer.WebServer.main()
generator.Scheduler.main()
diff --git a/src/main/scala/generator/GenerateNewsPaper.scala b/src/main/scala/generator/GenerateNewsPaper.scala
@@ -1,11 +1,65 @@
package generator
+import java.io.File
+import java.text.SimpleDateFormat
+import java.util.Date
+
+import com.github.tototoshi.csv.CSVReader
import com.typesafe.scalalogging.StrictLogging
import org.quartz.{Job, JobExecutionContext}
+import twitterCrawler.RestAPISingleton
+
+import scala.concurrent.Await
class GenerateNewsPaper extends Job with StrictLogging {
def execute(context: JobExecutionContext): Unit = {
//TODO parse CSV from yesterday and generate Feed
- logger.info("unimplemented")
+ val reader = this.getReader
+ val api = RestAPISingleton.getRestAPI
+ reader.foreach(line => {
+ val tweetID: Long = line.head.toLong
+ println(tweetID.toString)
+ //Get Tweet from id
+ val tweets = Await.result(api.getTweet(id = tweetID), scala.concurrent.duration.Duration.Inf)
+ val tweet = tweets.data
+ tweet.in_reply_to_status_id_str match {
+ case Some(value) => println("inReplyTo: ", value)
+ case None => println("inReplyTo: None")
+ }
+ tweet.extended_entities match {
+ case Some(value) => println("ExtendedEntities: ", value)
+ case None => println("ExtendedEntities: None")
+ }
+ val df = new SimpleDateFormat("dd.MM.yyyy HH:mm:ss")
+ println("Created at: ", df.format(tweet.created_at))
+ println("ID: ", tweet.id)
+ println("RetweetCount: ", tweet.retweet_count)
+ println("RetweetStatus: ", tweet.retweeted)
+ println("FavoriteCount: ", tweet.favorite_count)
+ println("Text:", tweet.text)
+ tweet.user match {
+ case Some(value) => println("User: ", value)
+ case None => println("User: None")
+ }
+ })
+ }
+
+ private def getReader: CSVReader = {
+ var reader: CSVReader = null
+ val yesterdayDate = this.yesterday
+ val yesterday = new SimpleDateFormat("dd_MM_yyyy").format(yesterdayDate)
+ val yesterdaysTweets = new File(s"data/tweets.$yesterday")
+ if (yesterdaysTweets.exists()) {
+ reader = CSVReader.open(yesterdaysTweets)
+ }
+ reader
+ }
+
+ import java.util.Calendar
+
+ private def yesterday: Date = {
+ val cal = Calendar.getInstance
+ cal.add(Calendar.DATE, -1)
+ cal.getTime
}
}
diff --git a/src/main/scala/generator/Scheduler.scala b/src/main/scala/generator/Scheduler.scala
@@ -7,6 +7,9 @@ import org.quartz.JobBuilder.newJob
import org.quartz.TriggerBuilder.newTrigger
import org.quartz.impl.StdSchedulerFactory
+import scala.concurrent.{Await, Future}
+import scala.concurrent.ExecutionContext.Implicits.global
+
object Scheduler {
val conf: Config = ConfigFactory.load()
@@ -27,6 +30,17 @@ object Scheduler {
// Tell quartz to schedule the job using our trigger
scheduler.scheduleJob(job, trigger)
+ /**
+ * Keep alive workaround
+ */
+ val waitFunc = Future {
+ while (true) {
+ Thread.sleep(1000)
+ }
+ }
+
+ Await.result(waitFunc, scala.concurrent.duration.Duration.Inf)
+
scheduler.shutdown()
} catch {
case se: SchedulerException =>
diff --git a/src/main/scala/twitterCrawler/StreamingApi.scala b/src/main/scala/twitterCrawler/StreamingApi.scala
@@ -17,6 +17,19 @@ import scala.collection.JavaConverters._
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.{Await, Future}
+
+object RestAPISingleton {
+ private var restAPI: TwitterRestClient = _
+ def getRestAPI: TwitterRestClient = {
+ this.restAPI
+ }
+
+ def setRestAPI(restAPI: TwitterRestClient): Unit = {
+ this.restAPI = restAPI
+ }
+
+}
+
/**
*
* StreamingAPI class is the connector for the Twitter Streaming Api
@@ -24,11 +37,10 @@ import scala.concurrent.{Await, Future}
* It mainly has the purpose to get Tweets of the users from the Lists and Hashtags defined inside the Config
*
* @param streamingClient holds the current Client for the Twitter Streaming API
- * @param restClient holds the current Client for the Twitter Rest API
*/
-class StreamingApi(val streamingClient: TwitterStreamingClient,
- val restClient: TwitterRestClient) extends StrictLogging {
+class StreamingApi(val streamingClient: TwitterStreamingClient) extends StrictLogging {
val conf: Config = ConfigFactory.load()
+ val restClient: TwitterRestClient = RestAPISingleton.getRestAPI
/**
* fetchTweets is a async function, that listens on Twitter's Streaming API for the defined Lists and Hastags
