Table of Contents
Why Scrape Job Boards?
Job market data is incredibly valuable for multiple industries. Scraping job boards enables powerful intelligence:
- Recruitment intelligence — Monitor competitor hiring and identify talent trends
- Salary benchmarking — Build comprehensive compensation databases
- Market research — Track demand for skills and technologies
- Lead generation — Identify companies actively hiring for sales outreach
- Investment analysis — Gauge company growth through hiring patterns
- Academic research — Study labor market dynamics and trends
Market Size
The global recruitment industry is worth over $500 billion. Companies that leverage job market data gain significant competitive advantages in talent acquisition and market positioning.
Data You Can Extract
From Job Listings
- Job title — Position name and level
- Company — Employer name and details
- Location — City, state, remote options
- Salary — Posted compensation ranges
- Description — Full job requirements and duties
- Posted date — When the job was listed
- Job type — Full-time, part-time, contract
- Experience level — Entry, mid, senior, executive
From Company Profiles
- Company size — Employee count ranges
- Industry — Business sector
- Headquarters — Location information
- Ratings — Employee satisfaction scores
- Open positions — Total active listings
Scraping LinkedIn Jobs
LinkedIn is the world's largest professional network with millions of job listings. Here's how to scrape their public job board:
Python
from playwright.sync_api import sync_playwright
import time
import random
class LinkedInJobScraper:
def __init__(self):
self.playwright = sync_playwright().start()
self.browser = self.playwright.chromium.launch(headless=True)
def search_jobs(self, keywords, location, num_pages=5):
jobs = []
page = self.browser.new_page()
for page_num in range(num_pages):
start = page_num * 25
url = f"https://www.linkedin.com/jobs/search/?keywords={keywords}&location={location}&start={start}"
page.goto(url, wait_until='networkidle')
time.sleep(random.uniform(2, 4))
# Scroll to load lazy content
for _ in range(3):
page.mouse.wheel(0, 1000)
time.sleep(0.5)
job_cards = page.locator('.base-card').all()
for card in job_cards:
try:
job = {
'title': card.locator('.base-search-card__title').text_content().strip(),
'company': card.locator('.base-search-card__subtitle').text_content().strip(),
'location': card.locator('.job-search-card__location').text_content().strip(),
'link': card.locator('a.base-card__full-link').get_attribute('href'),
'posted': card.locator('time').get_attribute('datetime'),
}
jobs.append(job)
except:
continue
page.close()
return jobs
def close(self):
self.browser.close()
self.playwright.stop()
# Usage
scraper = LinkedInJobScraper()
jobs = scraper.search_jobs('software engineer', 'San Francisco', num_pages=3)
scraper.close()LinkedIn Considerations
LinkedIn has aggressive anti-scraping measures. Stick to their public job board (no login required), use residential proxies, implement generous delays, and never attempt to scrape user profiles without authorization.
Scraping Indeed
Indeed aggregates jobs from across the web, making it ideal for comprehensive market coverage:
Python
import requests
from bs4 import BeautifulSoup
class IndeedScraper:
def __init__(self):
self.headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
'Accept-Language': 'en-US,en;q=0.9',
}
def search_jobs(self, query, location, pages=5):
jobs = []
for page in range(pages):
start = page * 10
url = f"https://www.indeed.com/jobs?q={query}&l={location}&start={start}"
response = requests.get(url, headers=self.headers)
soup = BeautifulSoup(response.text, 'html.parser')
job_cards = soup.select('.job_seen_beacon')
for card in job_cards:
title_elem = card.select_one('.jobTitle span')
company_elem = card.select_one('[data-testid="company-name"]')
location_elem = card.select_one('[data-testid="text-location"]')
salary_elem = card.select_one('.salary-snippet-container')
snippet_elem = card.select_one('.job-snippet')
job = {
'title': title_elem.text.strip() if title_elem else None,
'company': company_elem.text.strip() if company_elem else None,
'location': location_elem.text.strip() if location_elem else None,
'salary': salary_elem.text.strip() if salary_elem else None,
'description': snippet_elem.text.strip() if snippet_elem else None,
}
jobs.append(job)
return jobs
# Usage
scraper = IndeedScraper()
jobs = scraper.search_jobs('data scientist', 'New York', pages=3)Scraping Glassdoor
Glassdoor offers unique salary and company review data. Focus on publicly accessible content:
Python
from playwright.sync_api import sync_playwright
import json
class GlassdoorScraper:
def __init__(self):
self.playwright = sync_playwright().start()
self.browser = self.playwright.chromium.launch(headless=True)
def search_jobs(self, query, location):
jobs = []
page = self.browser.new_page()
# Glassdoor job search URL
url = f"https://www.glassdoor.com/Job/jobs.htm?sc.keyword={query}&locT=C&locId=1147401"
page.goto(url, wait_until='networkidle')
# Wait for job listings to load
page.wait_for_selector('[data-test="jobListing"]', timeout=10000)
job_cards = page.locator('[data-test="jobListing"]').all()
for card in job_cards:
try:
job = {
'title': card.locator('[data-test="job-title"]').text_content().strip(),
'company': card.locator('[data-test="emp-name"]').text_content().strip(),
'location': card.locator('[data-test="emp-location"]').text_content().strip(),
'salary': self._extract_salary(card),
'rating': self._extract_rating(card),
}
jobs.append(job)
except:
continue
page.close()
return jobs
def _extract_salary(self, card):
try:
return card.locator('[data-test="detailSalary"]').text_content().strip()
except:
return None
def _extract_rating(self, card):
try:
return card.locator('[data-test="rating"]').text_content().strip()
except:
return None
def close(self):
self.browser.close()
self.playwright.stop()Processing Job Data
Python
import re
from dataclasses import dataclass
from typing import Optional
@dataclass
class ProcessedJob:
title: str
company: str
location: str
salary_min: Optional[float]
salary_max: Optional[float]
job_type: Optional[str]
experience_level: Optional[str]
def parse_salary(salary_str):
"""Extract min/max salary from various formats."""
if not salary_str:
return None, None
# Remove currency symbols and normalize
cleaned = re.sub(r'[^\d\s\-kK]', '', salary_str)
# Handle "K" notation (e.g., "80K - 120K")
cleaned = re.sub(r'(\d+)[kK]', lambda m: str(int(m.group(1)) * 1000), cleaned)
numbers = re.findall(r'\d+', cleaned)
if len(numbers) >= 2:
return float(numbers[0]), float(numbers[1])
elif len(numbers) == 1:
return float(numbers[0]), float(numbers[0])
return None, None
def extract_experience_level(title):
"""Infer experience level from job title."""
title_lower = title.lower()
if any(word in title_lower for word in ['senior', 'sr.', 'lead', 'principal']):
return 'senior'
elif any(word in title_lower for word in ['junior', 'jr.', 'entry']):
return 'entry'
elif any(word in title_lower for word in ['manager', 'director', 'head', 'vp']):
return 'executive'
return 'mid'Business Use Cases
1. Recruitment Analytics Dashboard
Build dashboards showing hiring trends by industry, location, and role. Track which companies are expanding and which skills are in demand.
2. Compensation Benchmarking
Create salary databases to help companies benchmark compensation and help job seekers negotiate effectively.
3. Sales Lead Generation
Companies posting jobs are actively growing. Use job posting data to identify warm leads for B2B sales.
4. Talent Market Reports
Produce market intelligence reports on hiring trends, skill demands, and salary movements for HR consulting.
Need Job Market Intelligence?
Crawlix provides real-time job board data from LinkedIn, Indeed, Glassdoor and 50+ other sources with high reliability.
Get Job Data →