using System; using System.Collections.Generic; using System.Linq; using System.Text; using TweetSharp.Twitter; using TweetSharp.Twitter.Model; using TweetSharp.Twitter.Service; using TweetSharp.Service; using TweetSharp.Model; using TweetSharp.Twitter.Fluent; using TweetSharp.Extensions; using TweetSharp.Twitter.Extensions; using TweetSharp; using System.Diagnostics; using System.Xml.Serialization; using System.Xml; using System.Windows.Forms; using System.IO; namespace EchoChamber.Library { public class TwitterScraper { public static TweetSet Scrape(string twit) { List statusList = new List(); long lastTweetID = 0; TwitterService twitterService = EchoChamberHelper.GetAuthenticatedTwitter(); for (int page = 1; page < 20; page++) { try { IEnumerable result = twitterService.ListTweetsOnSpecifiedUserTimeline(twit, page, 200); if (result == null || result.Count() == 0) { break; } foreach (TwitterStatus status in result) { statusList.Add(status.Text); lastTweetID = Math.Max(status.Id, lastTweetID); ; } } catch (Exception ex) { MessageBox.Show(ex.ToString(), twit); break; } } TweetSet tweetSet = new TweetSet(); tweetSet.Tweets = statusList; tweetSet.LastTweetID = lastTweetID; return tweetSet; } public static TweetSet Scrape(string twit, long lastTweetID) { List statusList = new List(); TwitterService twitterService = EchoChamberHelper.GetAuthenticatedTwitter(); IEnumerable result = twitterService.ListTweetsOnSpecifiedUserTimelineSince(twit, lastTweetID, 200); if (result != null && result.Count() == 0) { foreach (TwitterStatus status in result) { statusList.Add(status.Text); lastTweetID = Math.Max(status.Id, lastTweetID); } } TweetSet tweetSet = new TweetSet(); tweetSet.Tweets = statusList; tweetSet.LastTweetID = lastTweetID; return tweetSet; } public static string[] GetTwits() { //Service calls fail. Hard code for time. return new string[]{"AndrewBreitbart", "ScottBrownMA", "TuckerCarlson", "MittRomney", "ewerickson", "gretawire", "hannityshow", "RepRonPaul", "EricCantor", "foxnews", "MicheleBachmann", "RedState", "marcorubio", "DanaPerino", "Bret_Baier", "GOPWhip", "oreillyfactor", "DinoRossiWA", "GovBrewer", "SarahPalinUSA", "CatoInstitute", "SenJohnMcCain", "GovMikeHuckabee", "glennbeck", "newtgingrich", "JimDeMint", "michellemalkin", "GOPLeader", "Whitman2010", "marklevinshow", "KarlRove", "Heritage", "Drudge_Report", "johnboehner", "seanhannity", "reasonmag", "CarlyforCA"}; /*//Only handles 100 users for now. Should be finefor Project needs. List twits = new List(); TwitterService twitterService = EchoChamberHelper.GetAuthenticatedTwitter(); TwitterCursorList twitList = twitterService.ListFriends(); twits.AddRange(twitList.Select(x => x.ScreenName)); return twits.ToArray();*/ } public static void UpdateTweets(string twit) { Dictionary stateTable = TwitterState.LoadState("twitterState.xml"); if (stateTable.ContainsKey(twit)) { TweetSet tweetSet = TwitterScraper.Scrape(twit, stateTable[twit].LastTweetID); stateTable[twit].LastTweetID = tweetSet.LastTweetID; string tweets = string.Join(Environment.NewLine, tweetSet.Tweets.ToArray()); //Refudiate the punctuators. tweets = EchoChamberHelper.SanitizeTweets(tweets); Indexer indexer = Indexer.Load(twit + ".xml"); indexer.Index(tweets); indexer.Save(twit + ".xml"); File.AppendAllText(twit + ".txt", tweets); } else { stateTable[twit] = new TwitterState() { Twit = twit, LastTweetID = 0 }; TweetSet tweetSet = TwitterScraper.Scrape(twit); stateTable[twit].LastTweetID = tweetSet.LastTweetID; string tweets = string.Join(Environment.NewLine, tweetSet.Tweets.ToArray()); //Refudiate the punctuators. tweets = EchoChamberHelper.SanitizeTweets(tweets); Indexer indexer = new Indexer(); indexer.Name = twit; indexer.Index(tweets); indexer.Save(twit + ".xml"); File.AppendAllText(twit + ".txt", tweets); } TwitterState.SaveState("twitterState.xml", stateTable); } /* //This Authorization Code won't work as-is. You'll have to insert your application keys here. //It then saves clientinfo.xml and authtoken.xml for later use. I'm not checking those files in because they're secret. public static string OAUTH_CONSUMER_KEY = "Insert Here"; public static string OAUTH_CONSUMER_SECRET = "Insert Here"; public static void Authorize() { TwitterClientInfo info = new TwitterClientInfo(); info.ConsumerKey = OAUTH_CONSUMER_KEY; info.ConsumerSecret = OAUTH_CONSUMER_SECRET; XmlSerializer infoSerializer = new XmlSerializer(typeof(TwitterClientInfo)); using (XmlWriter writer = XmlWriter.Create("clientinfo.xml")) { infoSerializer.Serialize(writer, info); } var service = new TwitterService(info); OAuthToken requestToken = service.GetRequestToken(); String AuthURL = service.GetAuthorizationUrl(requestToken); Process.Start(AuthURL); // Launch browser with authorization URL. MessageBox.Show("Answer the question in the browser window and copy the pin on the clipboard, click ok when ready"); String Pin = Clipboard.GetText(TextDataFormat.Text); OAuthToken accessToken = service.GetAccessToken(requestToken, Pin); service.AuthenticateWith(accessToken.Token, accessToken.TokenSecret); XmlSerializer authSerializer = new XmlSerializer(typeof(OAuthToken)); using (XmlWriter writer = XmlWriter.Create("authtoken.xml")) { authSerializer.Serialize(writer, accessToken); } if (service.Error == null) { service.SendTweet("Access Granted. Carry On."); } }*/ } public class TweetSet { public List Tweets { get; set; } public long LastTweetID { get; set; } } public class TwitterState { public string Twit { get; set; } public long LastTweetID { get; set; } public static Dictionary LoadState(string filename) { if (!File.Exists(filename)) { return new Dictionary(); } List stateList = EchoChamberHelper.Deserialize>(filename); Dictionary stateTable = new Dictionary(); stateList.ForEach(x => stateTable[x.Twit] = x); return stateTable; } public static void SaveState(string filename, Dictionary stateTable) { List stateList = stateTable.Values.ToList(); EchoChamberHelper.Serialize(filename, stateList); } } }