socialnetworknews-api

git clone git://archive.git.mtrnord.blog/SocialNetworkNews/socialnetworknews-api.git
Log | Files | Refs | LICENSE

twitter.go (7940B)

      1 package twitter
      2 
      3 import (
      4 	"encoding/csv"
      5 	"encoding/json"
      6 	"fmt"
      7 	"github.com/SocialNetworkNews/SocialNetworkNews_API/config"
      8 	"github.com/SocialNetworkNews/anaconda"
      9 	"github.com/dghubble/oauth1"
     10 	"github.com/dghubble/oauth1/twitter"
     11 	"net/url"
     12 	"os"
     13 	"path/filepath"
     14 	"strconv"
     15 	"strings"
     16 	"sync"
     17 	"time"
     18 )
     19 
     20 var apiOnce sync.Once
     21 var api *TwitterAPI
     22 
     23 type Tweets struct {
     24 	Tweets []Tweet `json:"tweets,omitempty"`
     25 }
     26 
     27 type Tweet struct {
     28 	Username                 string   `json:"username"`
     29 	UserID                   string   `json:"user_id"`
     30 	DisplayName              string   `json:"display_name"`
     31 	UserProfileLink          string   `json:"userprofile_link"`
     32 	RetweetByUsername        string   `json:"retweetby_username"`
     33 	RetweetByUserID          string   `json:"retweetby_user_id"`
     34 	RetweetByDisplayName     string   `json:"retweetby_display_name"`
     35 	RetweetByUserProfileLink string   `json:"retweetby_userprofile_link"`
     36 	TweetLink                string   `json:"tweet_link"`
     37 	Text                     string   `json:"text"`
     38 	IMGUrls                  []string `json:"image_urls"`
     39 	CreatedAt                string   `json:"created_at"`
     40 	Favorites                string   `json:"favorites"`
     41 	Retweets                 string   `json:"retweets"`
     42 	Retweet                  bool     `json:"retweet"`
     43 }
     44 
     45 type TwitterAPI struct {
     46 	api    *anaconda.TwitterApi
     47 	stream *anaconda.Stream
     48 }
     49 
     50 func NewTwitterAPIStruct() *TwitterAPI {
     51 	apiOnce.Do(func() {
     52 		api = &TwitterAPI{}
     53 	})
     54 	return api
     55 }
     56 
     57 func (t *TwitterAPI) getTokens(config *oauth1.Config) (string, string, error) {
     58 	requestToken, requestSecret, TokenErr := config.RequestToken()
     59 	if TokenErr != nil {
     60 		return "", "", TokenErr
     61 	}
     62 
     63 	authorizationURL, URLErr := config.AuthorizationURL(requestToken)
     64 	if URLErr != nil {
     65 		return "", "", URLErr
     66 	}
     67 
     68 	fmt.Println(authorizationURL.String())
     69 	fmt.Print("Enter PinCode: ")
     70 	var pin string
     71 	fmt.Scanln(&pin)
     72 
     73 	accessToken, accessSecret, ATErr := config.AccessToken(requestToken, requestSecret, pin)
     74 	if ATErr != nil {
     75 		return "", "", ATErr
     76 	}
     77 
     78 	return accessToken, accessSecret, nil
     79 
     80 }
     81 
     82 func (t *TwitterAPI) Login(consumerKey, consumerSecret string) (err error) {
     83 	oauthConfig := &oauth1.Config{
     84 		ConsumerKey:    consumerKey,
     85 		ConsumerSecret: consumerSecret,
     86 		Endpoint:       twitter.AuthorizeEndpoint,
     87 	}
     88 	accessToken, accessSecret, ATErr := t.getTokens(oauthConfig)
     89 	if ATErr != nil {
     90 		err = ATErr
     91 		return
     92 	}
     93 
     94 	anaconda.SetConsumerKey(consumerKey)
     95 	anaconda.SetConsumerSecret(consumerSecret)
     96 	t.api = anaconda.NewTwitterApi(accessToken, accessSecret)
     97 	return
     98 }
     99 
    100 func (t *TwitterAPI) StartListening(lists []string, hashtags []string) error {
    101 	fmt.Println("StartListening")
    102 	Alists, AListErr := t.getLists(lists)
    103 	if AListErr != nil {
    104 		return AListErr
    105 	}
    106 	usersS, MembersErr := t.getListMembersS(Alists)
    107 	if MembersErr != nil {
    108 		return MembersErr
    109 	}
    110 
    111 	hastagsS := strings.Join(hashtags, ",")
    112 
    113 	v := url.Values{}
    114 	v.Set("follow", usersS)
    115 	v.Set("track", hastagsS)
    116 
    117 	t.stream = t.api.PublicStreamFilter(v)
    118 	for tw := range t.stream.C {
    119 		switch v := tw.(type) {
    120 		case anaconda.Tweet:
    121 			currentTime := time.Now().Local()
    122 			t.writeCSV([]string{v.IdStr, currentTime.String()})
    123 			fmt.Printf("%-15s: %s\n", v.User.ScreenName, v.FullText)
    124 		default:
    125 			fmt.Println("Got some other Type: ", v)
    126 		}
    127 	}
    128 	return nil
    129 }
    130 
    131 func (t *TwitterAPI) writeCSV(tweet []string) error {
    132 	currentTime := time.Now().Local()
    133 	dataPath := filepath.Join(".", "data")
    134 	if _, err := os.Stat(dataPath); os.IsNotExist(err) {
    135 		os.Mkdir(dataPath, os.ModePerm)
    136 	}
    137 	filename := fmt.Sprintf("tweets_%s.csv", currentTime.Format("2006_01_02"))
    138 	filePath := config.ConfigPath()
    139 	dataFilePath := filepath.Join(filePath, "data", filename)
    140 
    141 	// write the file
    142 	f, err := os.OpenFile(dataFilePath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
    143 	if err != nil {
    144 		return err
    145 	}
    146 	w := csv.NewWriter(f)
    147 	w.Write(tweet)
    148 	w.Flush()
    149 	defer f.Close()
    150 	return nil
    151 }
    152 
    153 func (t *TwitterAPI) getLists(lists []string) ([]anaconda.List, error) {
    154 	var Alists []anaconda.List
    155 	for _, l := range lists {
    156 		s := strings.Split(l, "/")
    157 		v := url.Values{}
    158 		list, err := t.api.GetListBySlug(s[1], s[0], v)
    159 		if err != nil {
    160 			return nil, err
    161 		}
    162 		Alists = append(Alists, list)
    163 	}
    164 
    165 	return Alists, nil
    166 }
    167 
    168 func (t *TwitterAPI) getListMembersS(lists []anaconda.List) (string, error) {
    169 	var users []anaconda.User
    170 	var usersSS []string
    171 	for _, l := range lists {
    172 		v := url.Values{}
    173 		members, err := t.api.GetListMembers(l.Id, v)
    174 		if err != nil {
    175 			return "", err
    176 		}
    177 
    178 		users = append(users, members...)
    179 	}
    180 
    181 	for _, u := range users {
    182 		usersSS = append(usersSS, u.IdStr)
    183 	}
    184 
    185 	usersS := strings.Join(usersSS, ",")
    186 	return usersS, nil
    187 }
    188 
    189 func (t *TwitterAPI) GetTweets(tweets []int64) ([]byte, error) {
    190 	v := url.Values{}
    191 	v.Set("include_entities", "true")
    192 	retweets := make(map[string]bool)
    193 	tweetsSlice := make(map[string]bool)
    194 
    195 	var divided [][]int64
    196 
    197 	chunkSize := 99
    198 
    199 	for i := 0; i < len(tweets); i += chunkSize {
    200 		end := i + chunkSize
    201 
    202 		if end > len(tweets) {
    203 			end = len(tweets)
    204 		}
    205 
    206 		divided = append(divided, tweets[i:end])
    207 	}
    208 
    209 	var JTweets []Tweet
    210 
    211 	for _, chunk := range divided {
    212 		tweetsR, tweetErr := t.api.GetTweetsLookupByIds(chunk, v)
    213 		if tweetErr != nil {
    214 			return nil, tweetErr
    215 		}
    216 
    217 		for _, t := range tweetsR {
    218 			JT := Tweet{}
    219 			if tweetsSlice[t.IdStr] {
    220 				continue
    221 			}
    222 			if t.RetweetedStatus != nil {
    223 				JT.Retweet = true
    224 				if retweets[t.RetweetedStatus.IdStr] {
    225 					continue
    226 				} else {
    227 					retweets[t.RetweetedStatus.IdStr] = true
    228 				}
    229 			} else {
    230 				JT.Retweet = false
    231 			}
    232 			tweetsSlice[t.IdStr] = true
    233 			// If we got a retweet get the data of the original tweet
    234 			if JT.Retweet {
    235 				JT.Username = t.RetweetedStatus.User.ScreenName
    236 				JT.UserID = t.RetweetedStatus.User.IdStr
    237 
    238 				JT.DisplayName = t.RetweetedStatus.User.Name
    239 				JT.UserProfileLink = "https://twitter.com/" + t.RetweetedStatus.User.ScreenName
    240 
    241 				JT.RetweetByUsername = t.User.ScreenName
    242 				JT.RetweetByUserID = t.User.IdStr
    243 				JT.RetweetByDisplayName = t.User.Name
    244 				JT.RetweetByUserProfileLink = "https://twitter.com/" + t.User.ScreenName
    245 
    246 				JT.TweetLink = "https://twitter.com/" + t.RetweetedStatus.User.ScreenName + "/status/" + t.RetweetedStatus.IdStr
    247 				if t.RetweetedStatus.ExtendedTweet.FullText != "" {
    248 					JT.Text = t.RetweetedStatus.ExtendedTweet.FullText
    249 				} else if t.RetweetedStatus.FullText != "" {
    250 					JT.Text = t.RetweetedStatus.FullText
    251 				} else {
    252 					JT.Text = t.RetweetedStatus.Text
    253 				}
    254 
    255 				tweetTime, err := t.RetweetedStatus.CreatedAtTime()
    256 				if err != nil {
    257 					return nil, err
    258 				}
    259 				JT.CreatedAt = tweetTime.Format("02.01.2006")
    260 				JT.Favorites = strconv.Itoa(t.RetweetedStatus.FavoriteCount)
    261 				JT.Retweets = strconv.Itoa(t.RetweetedStatus.RetweetCount)
    262 
    263 				for _, i := range t.RetweetedStatus.Entities.Media {
    264 					if i.Type == "photo" {
    265 						JT.IMGUrls = append(JT.IMGUrls, i.Media_url_https)
    266 					}
    267 				}
    268 			} else {
    269 				JT.Username = t.User.ScreenName
    270 				JT.UserID = t.User.IdStr
    271 
    272 				JT.DisplayName = t.User.Name
    273 				JT.UserProfileLink = "https://twitter.com/" + t.User.ScreenName
    274 				JT.TweetLink = "https://twitter.com/" + t.User.ScreenName + "/status/" + t.IdStr
    275 				if t.ExtendedTweet.FullText != "" {
    276 					JT.Text = t.ExtendedTweet.FullText
    277 				} else if t.FullText != "" {
    278 					JT.Text = t.FullText
    279 				} else {
    280 					JT.Text = t.Text
    281 				}
    282 
    283 				tweetTime, err := t.CreatedAtTime()
    284 				if err != nil {
    285 					return nil, err
    286 				}
    287 				JT.CreatedAt = tweetTime.Format("02.01.2006")
    288 				JT.Favorites = strconv.Itoa(t.FavoriteCount)
    289 				JT.Retweets = strconv.Itoa(t.RetweetCount)
    290 
    291 				for _, i := range t.Entities.Media {
    292 					if i.Type == "photo" {
    293 						JT.IMGUrls = append(JT.IMGUrls, i.Media_url_https)
    294 					}
    295 				}
    296 			}
    297 
    298 			JTweets = append(JTweets, JT)
    299 		}
    300 	}
    301 
    302 	tweetsS := Tweets{}
    303 	tweetsS.Tweets = JTweets
    304 
    305 	tweetObject, err := json.Marshal(tweetsS)
    306 	if err != nil {
    307 		return nil, err
    308 	}
    309 
    310 	return tweetObject, nil
    311 }