commit 00e65ab3f73afe3890e88b2706aa142b438f75e2
Author: MTRNord <mtrnord1@gmail.com>
Date: Fri, 16 Feb 2018 19:33:00 +0100
Initial rewrite
Diffstat:
6 files changed, 404 insertions(+), 0 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,3 @@
+data/
+config.yaml
+.idea/
+\ No newline at end of file
diff --git a/api/webserver.go b/api/webserver.go
@@ -0,0 +1,69 @@
+package api
+
+import (
+ "github.com/gorilla/mux"
+ "net/http"
+ "fmt"
+ "github.com/SocialNetworkNews/SocialNetworkNews_API/twitter"
+ "os"
+ "path/filepath"
+ "time"
+ "encoding/csv"
+ "strconv"
+)
+
+func Yesterday(w http.ResponseWriter, r *http.Request) {
+ vars := mux.Vars(r)
+ uuid := vars["uuid"]
+ fmt.Println("UUID: ", uuid)
+
+ tweets, err := getTweets()
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ }
+ w.WriteHeader(http.StatusOK)
+ w.Write(tweets)
+}
+
+func getTweets() ([]byte, error) {
+ api := twitter.NewTwitterAPIStruct()
+
+ // open output file
+ currentTime := time.Now().Local()
+ currentTime = currentTime.AddDate(0, 0, -1)
+ filename := fmt.Sprintf("tweets_%s.csv", currentTime.Format("2006_01_02"))
+ dataFilePath := filepath.Join(".", "data", filename)
+
+ fo, err := os.Open(dataFilePath)
+ if err != nil {
+ return nil, err
+ }
+
+ // close fo on exit and check for its returned error
+ defer func() {
+ if err := fo.Close(); err != nil {
+ panic(err)
+ }
+ }()
+
+ r := csv.NewReader(fo)
+ data, readErr := r.ReadAll()
+ if readErr != nil {
+ return nil, readErr
+ }
+
+ var tweets []int64
+ for _, t := range data {
+ i, err := strconv.ParseInt(t[0], 10, 64)
+ if err != nil {
+ return nil, err
+ }
+ tweets = append(tweets, i)
+ }
+
+ tweetObject, err := api.GetTweets(tweets)
+ if err != nil {
+ return nil, err
+ }
+ return tweetObject, nil
+}
+\ No newline at end of file
diff --git a/config/config.go b/config/config.go
@@ -0,0 +1,46 @@
+package config
+
+import (
+ "gopkg.in/yaml.v2"
+ "io/ioutil"
+ "sync"
+)
+
+var gConfig *Config
+var configOnce sync.Once
+
+type Config struct {
+ TwitterConfig `yaml:"twitter"`
+}
+
+type TwitterConfig struct {
+ ConsumerKey string `yaml:"consumerKey"`
+ ConsumerSecret string `yaml:"consumerSecret"`
+ Lists []string `yaml:"lists,flow"`
+ Hashtags []string `yaml:"hashtags,flow"`
+}
+
+func GetConfig() (*Config, error) {
+ var gFerr error
+ configOnce.Do(func() {
+ config := &Config{}
+ b, err := ioutil.ReadFile("config.yaml") // just pass the file name
+ if err != nil {
+ gFerr = err
+ return
+ }
+
+ err = yaml.Unmarshal(b, config)
+ if err != nil {
+ gFerr = err
+ return
+ }
+
+ gConfig = config
+ })
+ if gFerr != nil {
+ return nil, gFerr
+ }
+
+ return gConfig, nil
+}
+\ No newline at end of file
diff --git a/config_example.yaml b/config_example.yaml
@@ -0,0 +1,5 @@
+twitter:
+ consumerKey: ""
+ consumerSecret: ""
+ hashtags: []
+ lists: []
+\ No newline at end of file
diff --git a/main.go b/main.go
@@ -0,0 +1,40 @@
+package main
+
+import (
+ "github.com/SocialNetworkNews/SocialNetworkNews_API/twitter"
+ "fmt"
+ "github.com/gorilla/mux"
+ "net/http"
+ web_api "github.com/SocialNetworkNews/SocialNetworkNews_API/api"
+ "log"
+ "github.com/rs/cors"
+ "github.com/SocialNetworkNews/SocialNetworkNews_API/config"
+)
+
+func main() {
+ api := twitter.NewTwitterAPIStruct()
+ configData, confErr := config.GetConfig()
+ if confErr != nil {
+ log.Fatal(confErr)
+ }
+
+ api.Login(configData.ConsumerKey, configData.ConsumerSecret)
+ fmt.Println("Logged in!")
+
+ r := mux.NewRouter()
+ p := r.PathPrefix("/paper/{uuid}").Subrouter()
+ p.HandleFunc("/yesterday", web_api.Yesterday).Methods("GET")
+ // cors.Default() setup the middleware with default options being
+ // all origins accepted with simple methods (GET, POST). See
+ // documentation below for more options.
+ handler := cors.Default().Handler(r)
+
+ go func() {
+ log.Fatal(http.ListenAndServe(":8000", handler))
+ }()
+
+ err := api.StartListening(configData.Lists,configData.Hashtags)
+ if err != nil {
+ fmt.Println(err)
+ }
+}
diff --git a/twitter/twitter.go b/twitter/twitter.go
@@ -0,0 +1,236 @@
+package twitter
+
+import (
+ "github.com/SocialNetworkNews/anaconda"
+ "github.com/dghubble/oauth1"
+ "github.com/dghubble/oauth1/twitter"
+ "fmt"
+ "net/url"
+ "strings"
+ "encoding/csv"
+ "os"
+ "time"
+ "path/filepath"
+ "sync"
+ "encoding/json"
+ "strconv"
+)
+
+var apiOnce sync.Once
+var api *TwitterAPI
+
+type Tweets struct {
+ Tweets []Tweet `json:"tweets,omitempty"`
+}
+
+type Tweet struct {
+ Username string `json:"username"`
+ UserID string `json:"user_id"`
+ DisplayName string `json:"display_name"`
+ UserProfileLink string `json:"userprofile_link"`
+ Text string `json:"text"`
+ IMGUrls []string `json:"image_urls"`
+ CreatedAt string `json:"created_at"`
+ Favorites string `json:"favorites"`
+ Retweets string `json:"retweets"`
+ Retweet bool `json:"retweet"`
+}
+
+type TwitterAPI struct {
+ api *anaconda.TwitterApi
+ stream *anaconda.Stream
+}
+
+func NewTwitterAPIStruct() *TwitterAPI {
+ apiOnce.Do(func() {
+ api = &TwitterAPI{}
+ })
+ return api
+}
+
+func(t *TwitterAPI) getTokens(config *oauth1.Config) (string ,string , error) {
+ requestToken, requestSecret, TokenErr := config.RequestToken()
+ if TokenErr != nil {
+ return "", "", TokenErr
+ }
+
+ authorizationURL, URLErr := config.AuthorizationURL(requestToken)
+ if URLErr != nil {
+ return "", "", URLErr
+ }
+
+ fmt.Println(authorizationURL.String())
+ fmt.Print("Enter PinCode: ")
+ var pin string
+ fmt.Scanln(&pin)
+
+ accessToken, accessSecret, ATErr := config.AccessToken(requestToken, requestSecret, pin)
+ if ATErr != nil {
+ return "", "", ATErr
+ }
+
+ return accessToken, accessSecret, nil
+
+}
+
+func(t *TwitterAPI) Login(consumerKey, consumerSecret string) (err error) {
+ oauthConfig := &oauth1.Config{
+ ConsumerKey: consumerKey,
+ ConsumerSecret: consumerSecret,
+ Endpoint: twitter.AuthorizeEndpoint,
+ }
+ accessToken, accessSecret, ATErr := t.getTokens(oauthConfig)
+ if ATErr != nil {
+ err = ATErr
+ return
+ }
+
+ anaconda.SetConsumerKey(consumerKey)
+ anaconda.SetConsumerSecret(consumerSecret)
+ t.api = anaconda.NewTwitterApi(accessToken, accessSecret)
+ return
+}
+
+func(t *TwitterAPI) StartListening(lists []string, hashtags []string) (error) {
+ fmt.Println("StartListening")
+ Alists, AListErr := t.getLists(lists)
+ if AListErr != nil {
+ return AListErr
+ }
+ usersS, MembersErr := t.getListMembersS(Alists)
+ if MembersErr != nil {
+ return MembersErr
+ }
+
+ hastagsS := strings.Join(hashtags, ",")
+
+ v := url.Values{}
+ v.Set("follow", usersS)
+ v.Set("track", hastagsS)
+
+ t.stream = t.api.PublicStreamFilter(v)
+ for tw := range t.stream.C {
+ switch v := tw.(type) {
+ case anaconda.Tweet:
+ currentTime := time.Now().Local()
+ t.writeCSV([]string{v.IdStr, currentTime.String()})
+ fmt.Printf("%-15s: %s\n", v.User.ScreenName, v.FullText)
+ default:
+ fmt.Println("Got some other Type: ", v)
+ }
+ }
+ return nil
+}
+
+func (t *TwitterAPI) writeCSV(tweet []string) (error) {
+ currentTime := time.Now().Local()
+ dataPath := filepath.Join(".", "data")
+ if _, err := os.Stat(dataPath); os.IsNotExist(err) {
+ os.Mkdir(dataPath, os.ModePerm)
+ }
+ filename := fmt.Sprintf("tweets_%s.csv", currentTime.Format("2006_01_02"))
+ dataFilePath := filepath.Join(".", "data", filename)
+
+ // write the file
+ f, err := os.OpenFile(dataFilePath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
+ if err != nil {
+ return err
+ }
+ w := csv.NewWriter(f)
+ w.Write(tweet)
+ w.Flush()
+ defer f.Close()
+ return nil
+}
+
+func (t *TwitterAPI) getLists(lists []string) ([]anaconda.List, error) {
+ var Alists []anaconda.List
+ for _, l := range lists {
+ s := strings.Split(l, "/")
+ v := url.Values{}
+ list, err := t.api.GetListBySlug(s[1], s[0], v)
+ if err != nil {
+ return nil, err
+ }
+ Alists = append(Alists, list)
+ }
+
+ return Alists, nil
+}
+
+func(t *TwitterAPI) getListMembersS(lists []anaconda.List) (string, error) {
+ var users []anaconda.User
+ var usersSS []string
+ for _, l := range lists {
+ v := url.Values{}
+ members, err := t.api.GetListMembers(l.Id, v)
+ if err != nil {
+ return "", err
+ }
+
+ users = append(users, members...)
+ }
+
+ for _, u := range users {
+ usersSS = append(usersSS, u.IdStr)
+ }
+
+ usersS := strings.Join(usersSS, ",")
+ return usersS, nil
+}
+
+func (t *TwitterAPI) GetTweets(tweets []int64) ([]byte, error) {
+ v := url.Values{}
+ v.Set("include_entities", "true")
+
+ tweetsR, tweetErr := t.api.GetTweetsLookupByIds(tweets, v)
+ if tweetErr != nil {
+ return nil, tweetErr
+ }
+ var JTweets []Tweet
+
+ for _, t := range tweetsR {
+ //fmt.Printf("%+v\n\n\n", t)
+ JT := Tweet{}
+ JT.Username = t.User.ScreenName
+ JT.UserID = t.User.IdStr
+ JT.DisplayName = t.User.Name
+ JT.UserProfileLink = "https://twitter.com/" + t.User.ScreenName
+ fmt.Println("ExtendedFull: ", t.ExtendedTweet.FullText)
+ fmt.Println("Full: ", t.FullText)
+ fmt.Println("Text: ", t.Text)
+ if t.ExtendedTweet.FullText != "" {
+ JT.Text = t.ExtendedTweet.FullText
+ } else if t.FullText != "" {
+ JT.Text = t.FullText
+ } else {
+ JT.Text = t.Text
+ }
+
+ tweetTime, err := t.CreatedAtTime()
+ if err != nil {
+ return nil, err
+ }
+ JT.CreatedAt = tweetTime.Format("02.01.2006")
+ JT.Favorites = strconv.Itoa(t.FavoriteCount)
+ JT.Retweets = strconv.Itoa(t.RetweetCount)
+ JT.Retweet = t.Retweeted
+ for _, i := range t.ExtendedEntities.Media {
+ if i.Type == "photo" {
+ JT.IMGUrls = append(JT.IMGUrls, i.Media_url_https)
+ }
+ }
+
+ JTweets = append(JTweets, JT)
+ }
+
+ tweetsS := Tweets{}
+ tweetsS.Tweets = JTweets
+
+ tweetObject, err := json.Marshal(tweetsS)
+ if err != nil {
+ return nil, err
+ }
+
+ return tweetObject, nil
+}
+\ No newline at end of file
