commit fe4f453f384db33cf2f40f630f1ec8049936bdf4
parent 08560c8df31e8a035ec23871aea1501714c7b8fa
Author: MTRNord <mtrnord1@gmail.com>
Date: Sun, 4 Feb 2018 12:15:02 +0100
Add native bundling support, basic Database support and logback config
Signed-off-by: MTRNord <mtrnord1@gmail.com>
Diffstat:
7 files changed, 88 insertions(+), 7 deletions(-)
diff --git a/build.sbt b/build.sbt
@@ -1,13 +1,32 @@
-name := "FreifunkNews"
-
+name := "SocialNetworkNews"
version := "1.0"
+// general package information (can be scoped to Windows)
+maintainer := "MTRNord <freifunknews@nordgedanken.blog>"
+packageSummary := "SocialNetworkNews"
+packageDescription := """A paper.li clone"""
+
+// wix build information
+wixProductId := "2d7d2fcd-bf87-4daf-99dc-279d1825f089"
+wixProductUpgradeId := "76b4ecc2-4d3e-4a59-bd63-e67e6cb29806"
scalaVersion := "2.12.3"
-resolvers += Resolver.sonatypeRepo("snapshots")
+// set the main class for packaging the main jar
+// 'run' will still auto-detect and prompt
+// change Compile to Test to set it for the test jar
+mainClass in (Compile, packageBin) := Some("Main")
+// set the main class for the main 'run' task
+// change Compile to Test to set it for 'test:run'
+mainClass in (Compile, run) := Some("Main")
+
+resolvers += Resolver.sonatypeRepo("snapshots")
libraryDependencies ++= Seq(
"com.danielasfregola" %% "twitter4s" % "5.5-SNAPSHOT",
"com.typesafe" % "config" % "1.3.2",
- "ch.qos.logback" % "logback-classic" % "1.1.9"
+ "ch.qos.logback" % "logback-classic" % "1.1.9",
+ "com.typesafe.slick" %% "slick" % "3.2.1",
+ "com.typesafe.slick" %% "slick-hikaricp" % "3.2.1"
)
+enablePlugins(JavaAppPackaging)
+
diff --git a/project/plugins.sbt b/project/plugins.sbt
@@ -0,0 +1,2 @@
+// for autoplugins
+addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.3")
+\ No newline at end of file
diff --git a/src/main/resources/application_example.conf b/src/main/resources/application_example.conf
@@ -14,6 +14,9 @@ twitter {
"MTRNord/freifunknews"
]
}
+db {
+ url = "jdbc:sqlite:./tweetdb.sqlite"
+}
akka {
}
diff --git a/src/main/resources/logback.xml b/src/main/resources/logback.xml
@@ -0,0 +1,11 @@
+<configuration>
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d{HH:mm:ss.SSS} SNN [%thread] %-5level %logger{36} - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <root level="info">
+ <appender-ref ref="STDOUT" />
+ </root>
+</configuration>
+\ No newline at end of file
diff --git a/src/main/scala/Main.scala b/src/main/scala/Main.scala
@@ -36,7 +36,7 @@ object Main extends App {
//sender.sendHelloWorld
val stream = new twitterCrawler.StreamingApi(streamingClient = streamingClient, restClient = restClient)
- stream.fetchHashtags
+ stream.fetchTweets
webServer.WebServer.main()
case Failure(err) => println(err.toString)
diff --git a/src/main/scala/database/Database.scala b/src/main/scala/database/Database.scala
@@ -0,0 +1,22 @@
+package database
+
+import java.sql.Date
+
+import slick.jdbc.SQLiteProfile.api._
+
+// Definition of the TWEETS table
+class Tweets(tag: Tag) extends Table[(Int, String, String, String, String, Date)](tag, "TWEETS") {
+ def id = column[Int]("ID", O.PrimaryKey) // This is the primary key column
+ def user = column[String]("USERNAME")
+ def text = column[String]("TEXT")
+ def url = column[String]("URL")
+ def interactions = column[String]("INTERACTIONS")
+ def indexingDate = column[Date]("ZIP")
+
+ // Every table needs a * projection with the same type as the table's type parameter
+ def * = (id, user, text, url, interactions, indexingDate)
+}
+
+class Database {
+
+}
diff --git a/src/main/scala/twitterCrawler/StreamingApi.scala b/src/main/scala/twitterCrawler/StreamingApi.scala
@@ -5,14 +5,18 @@ import com.danielasfregola.twitter4s.entities.{Tweet, User}
import com.danielasfregola.twitter4s.http.clients.streaming.TwitterStream
import com.danielasfregola.twitter4s.{TwitterRestClient, TwitterStreamingClient}
import com.typesafe.config.{Config, ConfigFactory}
+import org.joda.time.{DateTime, DateTimeZone}
import scala.collection.JavaConverters._
import scala.concurrent.ExecutionContext.Implicits.global
import scala.concurrent.{Await, Future}
+import slick.jdbc.SQLiteProfile.api._
+
+import scala.concurrent.duration.Duration
class StreamingApi (val streamingClient: TwitterStreamingClient, val restClient: TwitterRestClient) {
val conf: Config = ConfigFactory.load()
- def fetchHashtags: Future[TwitterStream] = {
+ def fetchTweets: Future[TwitterStream] = {
val trackedWords: Seq[String] = conf.getStringList("twitter.trackedWords").asScala
val trackedLists: List[String] = conf.getStringList("twitter.lists").asScala.toList
val trackedUsers: Seq[Long] = Await.result(this.getListUsers(trackedLists), scala.concurrent.duration.Duration.Inf)
@@ -21,11 +25,29 @@ class StreamingApi (val streamingClient: TwitterStreamingClient, val restClient:
s"And with tracked Users: $trackedUsers")
streamingClient.filterStatuses(tracks = trackedWords, follow = trackedUsers) {
- case tweet: Tweet => println(tweet.text)
+ case tweet: Tweet =>
+ println(tweet.text)
+ this.saveToDatabase(tweet)
case disconnect: DisconnectMessage => println("Disconnect: ", disconnect.disconnect.reason)
}
}
+ private def saveToDatabase(tweet: Tweet): Unit = {
+ val db = Database.forConfig("db")
+ try {
+ val tweets = TableQuery[database.Tweets]
+ val currentDate = new java.sql.Date(DateTime.now(DateTimeZone.UTC).getMillis)
+ val sqlSetup = DBIO.seq(
+ // Create the tables, including primary and foreign keys
+ tweets.schema.create,
+
+ tweets += (tweet.id.toInt, "", "", "", "", currentDate)
+ )
+ val setupFuture = db.run(sqlSetup)
+ Await.result(setupFuture, Duration.Inf)
+ } finally db.close()
+ }
+
private def getListUsers(trackedLists: List[String]): Future[Seq[Long]] = {
Future {
var trackedUsers: Seq[Long] = Seq()
