Table of Contents
Why Social Media Data Matters
Social media generates billions of data points daily. For marketers, this data reveals:
- Brand perception — What people really think about your brand
- Competitor intelligence — Track competitor campaigns and engagement
- Trend identification — Spot emerging trends before they peak
- Influencer discovery — Find authentic voices in your niche
- Content insights — Learn what content resonates with audiences
- Crisis detection — Early warning system for PR issues
Market Value
The social media analytics market is worth over $8 billion. Companies leveraging social data report 25% faster campaign optimization and 30% improvement in customer sentiment tracking.
Types of Data You Can Collect
Post-Level Data
- Content — Text, images, videos, links
- Engagement — Likes, comments, shares, saves
- Timing — Post date, time, frequency
- Hashtags — Tags used in posts
- Mentions — @mentions of accounts
Profile-Level Data
- Follower count — Audience size over time
- Following count — Network connections
- Bio information — Profile descriptions
- Verification status — Blue check presence
- Post frequency — Activity levels
Engagement Metrics
- Engagement rate — Interactions / Followers
- Reach estimates — Potential audience
- Virality — Share/repost rates
- Comment sentiment — Positive vs negative
Platform-by-Platform Guide
Twitter/X Scraping
Twitter offers rich public data through search and user timelines:
Python
from playwright.sync_api import sync_playwright
import time
class TwitterScraper:
def __init__(self):
self.playwright = sync_playwright().start()
self.browser = self.playwright.chromium.launch(headless=True)
def search_tweets(self, query, max_tweets=100):
tweets = []
page = self.browser.new_page()
url = f"https://twitter.com/search?q={query}&src=typed_query&f=live"
page.goto(url)
# Scroll to load tweets
while len(tweets) < max_tweets:
page.mouse.wheel(0, 2000)
time.sleep(2)
tweet_elements = page.locator('[data-testid="tweet"]').all()
for elem in tweet_elements:
try:
tweet = {
'text': elem.locator('[data-testid="tweetText"]').text_content(),
'author': elem.locator('[data-testid="User-Name"]').text_content(),
'likes': self._parse_count(elem, 'like'),
'retweets': self._parse_count(elem, 'retweet'),
'replies': self._parse_count(elem, 'reply'),
}
if tweet not in tweets:
tweets.append(tweet)
except:
continue
if len(tweets) >= max_tweets:
break
page.close()
return tweets[:max_tweets]
def _parse_count(self, elem, action):
try:
count_elem = elem.locator(f'[data-testid="{action}"] span').first
return count_elem.text_content() if count_elem.count() > 0 else '0'
except:
return '0'Instagram Scraping
Instagram requires careful handling due to aggressive anti-bot measures:
Python
class InstagramScraper:
def scrape_profile(self, username):
page = self.browser.new_page()
url = f"https://www.instagram.com/{username}/"
page.goto(url, wait_until='networkidle')
time.sleep(3) # Wait for dynamic content
profile = {
'username': username,
'followers': self._get_stat(page, 'followers'),
'following': self._get_stat(page, 'following'),
'posts': self._get_stat(page, 'posts'),
'bio': page.locator('header section > div:nth-child(3)').text_content(),
}
# Get recent posts
posts = []
post_links = page.locator('article a').all()[:12]
for link in post_links:
href = link.get_attribute('href')
posts.append({'url': f"https://instagram.com{href}"})
profile['recent_posts'] = posts
page.close()
return profile
def scrape_hashtag(self, hashtag):
page = self.browser.new_page()
url = f"https://www.instagram.com/explore/tags/{hashtag}/"
page.goto(url, wait_until='networkidle')
posts = []
post_elements = page.locator('article a').all()[:30]
for elem in post_elements:
posts.append({
'url': 'https://instagram.com' + elem.get_attribute('href'),
'hashtag': hashtag
})
page.close()
return postsTikTok Scraping
TikTok's explosive growth makes it essential for trend monitoring:
Python
class TikTokScraper:
def search_videos(self, keyword, max_videos=50):
videos = []
page = self.browser.new_page()
url = f"https://www.tiktok.com/search?q={keyword}"
page.goto(url)
# Scroll to load more videos
for _ in range(5):
page.mouse.wheel(0, 3000)
time.sleep(2)
video_cards = page.locator('[data-e2e="search_top-item"]').all()
for card in video_cards[:max_videos]:
try:
video = {
'description': card.locator('[data-e2e="search-card-desc"]').text_content(),
'author': card.locator('[data-e2e="search-card-user-unique-id"]').text_content(),
'likes': card.locator('[data-e2e="search-card-like-container"]').text_content(),
'link': card.locator('a').first.get_attribute('href'),
}
videos.append(video)
except:
continue
page.close()
return videosSentiment Analysis
Combine scraped data with NLP for sentiment insights:
Python
from textblob import TextBlob
from collections import Counter
def analyze_sentiment(posts):
"""Analyze sentiment of social media posts."""
results = {
'positive': 0,
'negative': 0,
'neutral': 0,
'posts': []
}
for post in posts:
text = post.get('text', '')
analysis = TextBlob(text)
# Classify sentiment
if analysis.sentiment.polarity > 0.1:
sentiment = 'positive'
elif analysis.sentiment.polarity < -0.1:
sentiment = 'negative'
else:
sentiment = 'neutral'
results[sentiment] += 1
results['posts'].append({
**post,
'sentiment': sentiment,
'polarity': analysis.sentiment.polarity
})
# Calculate percentages
total = len(posts)
results['positive_pct'] = results['positive'] / total * 100
results['negative_pct'] = results['negative'] / total * 100
results['neutral_pct'] = results['neutral'] / total * 100
return resultsInfluencer Research
Find and evaluate influencers in your niche:
Python
def calculate_engagement_rate(profile, posts):
"""Calculate true engagement rate for an influencer."""
total_engagement = sum(
post.get('likes', 0) + post.get('comments', 0)
for post in posts
)
avg_engagement = total_engagement / len(posts)
followers = profile.get('followers', 1)
engagement_rate = (avg_engagement / followers) * 100
return engagement_rate
def evaluate_influencer(profile, posts):
"""Score an influencer for campaign fit."""
return {
'username': profile['username'],
'followers': profile['followers'],
'engagement_rate': calculate_engagement_rate(profile, posts),
'post_frequency': len(posts) / 30, # posts per day
'avg_likes': sum(p.get('likes', 0) for p in posts) / len(posts),
'authenticity_score': estimate_authenticity(profile, posts),
}
def estimate_authenticity(profile, posts):
"""Estimate if followers are real based on engagement patterns."""
followers = profile.get('followers', 0)
avg_likes = sum(p.get('likes', 0) for p in posts) / len(posts)
# Suspicious if very low engagement for follower count
expected_min_engagement = followers * 0.01 # 1% minimum
if avg_likes < expected_min_engagement:
return 'low'
elif avg_likes > followers * 0.1: # Over 10% is great
return 'high'
return 'medium'Brand Monitoring
Set up automated brand mention tracking:
Python
class BrandMonitor:
def __init__(self, brand_keywords, competitors):
self.brand_keywords = brand_keywords
self.competitors = competitors
self.scrapers = {
'twitter': TwitterScraper(),
'instagram': InstagramScraper(),
}
def collect_mentions(self):
"""Collect all mentions across platforms."""
mentions = []
for keyword in self.brand_keywords:
# Twitter mentions
tweets = self.scrapers['twitter'].search_tweets(keyword)
for tweet in tweets:
tweet['platform'] = 'twitter'
tweet['keyword'] = keyword
mentions.append(tweet)
return mentions
def generate_report(self, mentions):
"""Generate brand monitoring report."""
sentiment_results = analyze_sentiment(mentions)
report = {
'total_mentions': len(mentions),
'sentiment_breakdown': {
'positive': sentiment_results['positive_pct'],
'negative': sentiment_results['negative_pct'],
'neutral': sentiment_results['neutral_pct'],
},
'top_positive': sorted(
[m for m in mentions if m.get('sentiment') == 'positive'],
key=lambda x: x.get('likes', 0),
reverse=True
)[:10],
'alerts': [
m for m in mentions
if m.get('sentiment') == 'negative' and m.get('likes', 0) > 100
],
}
return reportEthical Considerations
Important Guidelines
Social media scraping requires careful ethical consideration. Always: respect platform ToS, only collect public data, never access private profiles, implement rate limiting, don't collect personal identifiable information, and consider GDPR/CCPA compliance.
Best Practices
- Rate limiting — Don't overwhelm platforms with requests
- Data minimization — Only collect what you need
- No PII — Avoid collecting personal information
- Transparency — Be clear about how you use data
- Official APIs — Use when available and sufficient
Need Social Media Intelligence?
Crawlix provides compliant social media data collection with sentiment analysis and trend detection built-in.
Get Social Data →