commit 21538537be165f0de0554db9d70d8b411328e55d
parent 62c1fd2581dac81a35b33db7206aeee6ce09ab9d
Author: MTRNord <mtrnord1@gmail.com>
Date: Sat, 17 Feb 2018 12:13:11 +0100
Split requests into chunks
Diffstat:
4 files changed, 89 insertions(+), 75 deletions(-)
diff --git a/api/webserver.go b/api/webserver.go
@@ -1,15 +1,15 @@
package api
import (
- "github.com/gorilla/mux"
- "net/http"
+ "encoding/csv"
"fmt"
"github.com/SocialNetworkNews/SocialNetworkNews_API/twitter"
+ "github.com/gorilla/mux"
+ "net/http"
"os"
"path/filepath"
- "time"
- "encoding/csv"
"strconv"
+ "time"
)
func Yesterday(w http.ResponseWriter, r *http.Request) {
@@ -53,7 +53,7 @@ func getTweets() ([]byte, error) {
}
var tweets []int64
- for _, t := range data {
+ for _, t := range data {
i, err := strconv.ParseInt(t[0], 10, 64)
if err != nil {
return nil, err
@@ -66,4 +66,4 @@ func getTweets() ([]byte, error) {
return nil, err
}
return tweetObject, nil
-}
-\ No newline at end of file
+}
diff --git a/config/config.go b/config/config.go
@@ -14,10 +14,10 @@ type Config struct {
}
type TwitterConfig struct {
- ConsumerKey string `yaml:"consumerKey"`
- ConsumerSecret string `yaml:"consumerSecret"`
- Lists []string `yaml:"lists,flow"`
- Hashtags []string `yaml:"hashtags,flow"`
+ ConsumerKey string `yaml:"consumerKey"`
+ ConsumerSecret string `yaml:"consumerSecret"`
+ Lists []string `yaml:"lists,flow"`
+ Hashtags []string `yaml:"hashtags,flow"`
}
func GetConfig() (*Config, error) {
@@ -43,4 +43,4 @@ func GetConfig() (*Config, error) {
}
return gConfig, nil
-}
-\ No newline at end of file
+}
diff --git a/main.go b/main.go
@@ -1,14 +1,14 @@
package main
import (
- "github.com/SocialNetworkNews/SocialNetworkNews_API/twitter"
"fmt"
- "github.com/gorilla/mux"
- "net/http"
web_api "github.com/SocialNetworkNews/SocialNetworkNews_API/api"
- "log"
- "github.com/rs/cors"
"github.com/SocialNetworkNews/SocialNetworkNews_API/config"
+ "github.com/SocialNetworkNews/SocialNetworkNews_API/twitter"
+ "github.com/gorilla/mux"
+ "github.com/rs/cors"
+ "log"
+ "net/http"
)
func main() {
@@ -33,7 +33,7 @@ func main() {
log.Fatal(http.ListenAndServe(":8000", handler))
}()
- err := api.StartListening(configData.Lists,configData.Hashtags)
+ err := api.StartListening(configData.Lists, configData.Hashtags)
if err != nil {
fmt.Println(err)
}
diff --git a/twitter/twitter.go b/twitter/twitter.go
@@ -1,19 +1,19 @@
package twitter
import (
+ "encoding/csv"
+ "encoding/json"
+ "fmt"
"github.com/SocialNetworkNews/anaconda"
"github.com/dghubble/oauth1"
"github.com/dghubble/oauth1/twitter"
- "fmt"
"net/url"
- "strings"
- "encoding/csv"
"os"
- "time"
"path/filepath"
- "sync"
- "encoding/json"
"strconv"
+ "strings"
+ "sync"
+ "time"
)
var apiOnce sync.Once
@@ -24,20 +24,20 @@ type Tweets struct {
}
type Tweet struct {
- Username string `json:"username"`
- UserID string `json:"user_id"`
- DisplayName string `json:"display_name"`
- UserProfileLink string `json:"userprofile_link"`
- Text string `json:"text"`
- IMGUrls []string `json:"image_urls"`
- CreatedAt string `json:"created_at"`
- Favorites string `json:"favorites"`
- Retweets string `json:"retweets"`
- Retweet bool `json:"retweet"`
+ Username string `json:"username"`
+ UserID string `json:"user_id"`
+ DisplayName string `json:"display_name"`
+ UserProfileLink string `json:"userprofile_link"`
+ Text string `json:"text"`
+ IMGUrls []string `json:"image_urls"`
+ CreatedAt string `json:"created_at"`
+ Favorites string `json:"favorites"`
+ Retweets string `json:"retweets"`
+ Retweet bool `json:"retweet"`
}
type TwitterAPI struct {
- api *anaconda.TwitterApi
+ api *anaconda.TwitterApi
stream *anaconda.Stream
}
@@ -48,7 +48,7 @@ func NewTwitterAPIStruct() *TwitterAPI {
return api
}
-func(t *TwitterAPI) getTokens(config *oauth1.Config) (string ,string , error) {
+func (t *TwitterAPI) getTokens(config *oauth1.Config) (string, string, error) {
requestToken, requestSecret, TokenErr := config.RequestToken()
if TokenErr != nil {
return "", "", TokenErr
@@ -73,7 +73,7 @@ func(t *TwitterAPI) getTokens(config *oauth1.Config) (string ,string , error) {
}
-func(t *TwitterAPI) Login(consumerKey, consumerSecret string) (err error) {
+func (t *TwitterAPI) Login(consumerKey, consumerSecret string) (err error) {
oauthConfig := &oauth1.Config{
ConsumerKey: consumerKey,
ConsumerSecret: consumerSecret,
@@ -91,7 +91,7 @@ func(t *TwitterAPI) Login(consumerKey, consumerSecret string) (err error) {
return
}
-func(t *TwitterAPI) StartListening(lists []string, hashtags []string) (error) {
+func (t *TwitterAPI) StartListening(lists []string, hashtags []string) error {
fmt.Println("StartListening")
Alists, AListErr := t.getLists(lists)
if AListErr != nil {
@@ -122,7 +122,7 @@ func(t *TwitterAPI) StartListening(lists []string, hashtags []string) (error) {
return nil
}
-func (t *TwitterAPI) writeCSV(tweet []string) (error) {
+func (t *TwitterAPI) writeCSV(tweet []string) error {
currentTime := time.Now().Local()
dataPath := filepath.Join(".", "data")
if _, err := os.Stat(dataPath); os.IsNotExist(err) {
@@ -158,7 +158,7 @@ func (t *TwitterAPI) getLists(lists []string) ([]anaconda.List, error) {
return Alists, nil
}
-func(t *TwitterAPI) getListMembersS(lists []anaconda.List) (string, error) {
+func (t *TwitterAPI) getListMembersS(lists []anaconda.List) (string, error) {
var users []anaconda.User
var usersSS []string
for _, l := range lists {
@@ -183,45 +183,62 @@ func (t *TwitterAPI) GetTweets(tweets []int64) ([]byte, error) {
v := url.Values{}
v.Set("include_entities", "true")
- tweetsR, tweetErr := t.api.GetTweetsLookupByIds(tweets, v)
- if tweetErr != nil {
- return nil, tweetErr
+ var divided [][]int64
+
+ chunkSize := 99
+
+ for i := 0; i < len(tweets); i += chunkSize {
+ end := i + chunkSize
+
+ if end > len(tweets) {
+ end = len(tweets)
+ }
+
+ divided = append(divided, tweets[i:end])
}
+
var JTweets []Tweet
- for _, t := range tweetsR {
- //fmt.Printf("%+v\n\n\n", t)
- JT := Tweet{}
- JT.Username = t.User.ScreenName
- JT.UserID = t.User.IdStr
- JT.DisplayName = t.User.Name
- JT.UserProfileLink = "https://twitter.com/" + t.User.ScreenName
- fmt.Println("ExtendedFull: ", t.ExtendedTweet.FullText)
- fmt.Println("Full: ", t.FullText)
- fmt.Println("Text: ", t.Text)
- if t.ExtendedTweet.FullText != "" {
- JT.Text = t.ExtendedTweet.FullText
- } else if t.FullText != "" {
- JT.Text = t.FullText
- } else {
- JT.Text = t.Text
+ for _, chunk := range divided {
+ tweetsR, tweetErr := t.api.GetTweetsLookupByIds(chunk, v)
+ if tweetErr != nil {
+ return nil, tweetErr
}
- tweetTime, err := t.CreatedAtTime()
- if err != nil {
- return nil, err
- }
- JT.CreatedAt = tweetTime.Format("02.01.2006")
- JT.Favorites = strconv.Itoa(t.FavoriteCount)
- JT.Retweets = strconv.Itoa(t.RetweetCount)
- JT.Retweet = t.Retweeted
- for _, i := range t.ExtendedEntities.Media {
- if i.Type == "photo" {
- JT.IMGUrls = append(JT.IMGUrls, i.Media_url_https)
+ for _, t := range tweetsR {
+ //fmt.Printf("%+v\n\n\n", t)
+ JT := Tweet{}
+ JT.Username = t.User.ScreenName
+ JT.UserID = t.User.IdStr
+ JT.DisplayName = t.User.Name
+ JT.UserProfileLink = "https://twitter.com/" + t.User.ScreenName
+ fmt.Println("ExtendedFull: ", t.ExtendedTweet.FullText)
+ fmt.Println("Full: ", t.FullText)
+ fmt.Println("Text: ", t.Text)
+ if t.ExtendedTweet.FullText != "" {
+ JT.Text = t.ExtendedTweet.FullText
+ } else if t.FullText != "" {
+ JT.Text = t.FullText
+ } else {
+ JT.Text = t.Text
+ }
+
+ tweetTime, err := t.CreatedAtTime()
+ if err != nil {
+ return nil, err
+ }
+ JT.CreatedAt = tweetTime.Format("02.01.2006")
+ JT.Favorites = strconv.Itoa(t.FavoriteCount)
+ JT.Retweets = strconv.Itoa(t.RetweetCount)
+ JT.Retweet = t.Retweeted
+ for _, i := range t.ExtendedEntities.Media {
+ if i.Type == "photo" {
+ JT.IMGUrls = append(JT.IMGUrls, i.Media_url_https)
+ }
}
- }
- JTweets = append(JTweets, JT)
+ JTweets = append(JTweets, JT)
+ }
}
tweetsS := Tweets{}
@@ -233,4 +250,4 @@ func (t *TwitterAPI) GetTweets(tweets []int64) ([]byte, error) {
}
return tweetObject, nil
-}
-\ No newline at end of file
+}
