How to Automate Website Tasks with Python: Complete Guide 2026

💡 Real example: A recruitment manager in Toronto spent 90 minutes every Monday copying candidate data from a job portal into their internal tracking sheet. A 40-line Python script now logs into the portal, pulls the new applicant data, and updates the sheet automatically every Monday at 7 AM — before she even gets to her desk.

Feature	Requests + BS4	Selenium	Playwright
Controls real browser	No	Yes ✓	Yes ✓
Handles JavaScript	No ✗	Yes ✓	Yes ✓
Speed	⚡ Very fast	Moderate	Fast ✓
Auto-wait for elements	—	Manual ✗	Built-in ✓
Community & tutorials	Large ✓	Very large ✓	Growing
Best for	Static HTML scraping	Complex browser tasks	Modern JS-heavy sites

✅ Quick decision guide:
— Site is mostly static HTML? → Requests + BeautifulSoup
— Site uses JavaScript and you need a real browser? → Playwright (new projects) or Selenium (existing projects)
— You need to log in and navigate a member area? → Playwright or Selenium

# For static page scraping
pip install requests beautifulsoup4

# For browser automation with Playwright (recommended for new projects)
pip install playwright
playwright install chromium   # downloads the browser — run once

# For browser automation with Selenium
pip install selenium webdriver-manager

# Useful extras
pip install lxml               # faster HTML parser for BeautifulSoup
pip install python-dotenv      # store credentials in .env files

"""
Selenium basics: navigate, find elements, click, fill forms.
pip install selenium webdriver-manager
"""
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager

# --- Launch Chrome automatically (no manual driver download needed) ---
options = webdriver.ChromeOptions()
# options.add_argument("--headless")  # uncomment to run without visible window
driver = webdriver.Chrome(
    service=Service(ChromeDriverManager().install()),
    options=options
)
wait = WebDriverWait(driver, timeout=10)

# --- Navigate to a page ---
driver.get("https://example.com/search")

# --- Find elements ---
# By CSS selector (most reliable)
search_box = driver.find_element(By.CSS_SELECTOR, "input[name='q']")

# By ID
submit_btn = driver.find_element(By.ID, "submit-button")

# By text content
link = driver.find_element(By.LINK_TEXT, "Download Report")

# --- Interact ---
search_box.clear()
search_box.send_keys("python automation")
submit_btn.click()

# --- Wait for a result to appear ---
results = wait.until(
    EC.presence_of_all_elements_located((By.CSS_SELECTOR, ".result-item"))
)
for item in results[:5]:
    print(item.text)

# --- Take a screenshot ---
driver.save_screenshot("result.png")

driver.quit()

Pro tip: Always use WebDriverWait instead of time.sleep(). It waits only as long as needed and moves on the moment the element is found — making your scripts faster and more reliable.

"""
Playwright basics: navigate, fill forms, click, scrape.
pip install playwright && playwright install chromium
"""
from playwright.sync_api import sync_playwright

with sync_playwright() as p:
    # Launch Chromium (also supports Firefox, WebKit)
    browser = p.chromium.launch(headless=False)  # headless=True = no window
    page    = browser.new_page()

    # Navigate
    page.goto("https://example.com/search")

    # Fill input field (auto-waits until the element is ready)
    page.fill("input[name='q']", "python automation")

    # Click a button
    page.click("button[type='submit']")

    # Wait for navigation to complete
    page.wait_for_load_state("networkidle")

    # Get text from all matching elements
    results = page.locator(".result-title").all_text_contents()
    for title in results[:5]:
        print(title)

    # Select from a dropdown
    page.select_option("select#category", label="Technology")

    # Check a checkbox
    page.check("input#agree-terms")

    # Screenshot
    page.screenshot(path="screenshot.png", full_page=True)

    browser.close()

Playwright Codegen: Run playwright codegen https://example.com in your terminal. A browser opens — you click around manually, and Playwright automatically generates the Python code for every action. Perfect for getting started quickly.

"""
Scrape a table of job listings from a static HTML page.
pip install requests beautifulsoup4 lxml
"""
import csv
import requests
from bs4 import BeautifulSoup

URL     = "https://example-jobs.com/listings"
HEADERS = {"User-Agent": "Mozilla/5.0 (compatible; research-bot/1.0)"}

response = requests.get(URL, headers=HEADERS, timeout=10)
response.raise_for_status()  # raise error if request failed

soup = BeautifulSoup(response.text, "lxml")
jobs = []

for card in soup.select(".job-card"):
    title    = card.select_one(".job-title")
    company  = card.select_one(".company-name")
    location = card.select_one(".location")
    link     = card.select_one("a[href]")

    jobs.append({
        "title":    title.get_text(strip=True)    if title    else "",
        "company":  company.get_text(strip=True)  if company  else "",
        "location": location.get_text(strip=True) if location else "",
        "url":      link["href"]                   if link     else "",
    })

# Save to CSV
with open("jobs.csv", "w", newline="", encoding="utf-8") as f:
    writer = csv.DictWriter(f, fieldnames=["title", "company", "location", "url"])
    writer.writeheader()
    writer.writerows(jobs)

print(f"✅ Scraped {len(jobs)} jobs → saved to jobs.csv")

# Find first matching element
soup.find("h1")
soup.find("div", class_="price")
soup.find(id="main-content")

# Find all matching elements
soup.find_all("a")                      # all links
soup.select(".product-card")           # CSS class selector
soup.select("table tbody tr")          # nested selector
soup.select("a[href^='https']")        # links starting with https

# Extract data
element.get_text(strip=True)            # clean text content
element["href"]                        # attribute value
element.get("src", "")                 # safe attribute get

"""
Step 1: Log in once and save the session to a file.
Next time, load the session instead of logging in again.
"""
import os
from playwright.sync_api import sync_playwright
from dotenv import load_dotenv

load_dotenv()  # reads USERNAME and PASSWORD from .env file

SESSION_FILE = "session.json"

def login_and_save_session():
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=False)
        context = browser.new_context()
        page    = context.new_page()

        page.goto("https://example.com/login")

        # Fill login form
        page.fill("input[name='email']",    os.getenv("USERNAME"))
        page.fill("input[name='password']", os.getenv("PASSWORD"))
        page.click("button[type='submit']")

        # Wait until redirected to the dashboard
        page.wait_for_url("**/dashboard**")
        print("✅ Logged in successfully.")

        # Save cookies & storage state
        context.storage_state(path=SESSION_FILE)
        print(f"💾 Session saved to {SESSION_FILE}")
        browser.close()

def run_with_saved_session():
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        # Load the saved session — already logged in!
        context = browser.new_context(storage_state=SESSION_FILE)
        page    = context.new_page()

        page.goto("https://example.com/dashboard/reports")
        print(page.title())   # → "Reports | Dashboard"

        # Do your work here...
        browser.close()

# First run: login and save
if not os.path.exists(SESSION_FILE):
    login_and_save_session()
else:
    run_with_saved_session()

⚠️ Never hardcode credentials. Create a .env file with USERNAME=you@email.com and PASSWORD=yourpassword, add .env to your .gitignore, and load it with python-dotenv. Never commit credentials to Git.

import requests
from pathlib import Path

def download_file(url: str, save_path: str) -> None:
    # stream=True handles large files without loading all into memory
    response = requests.get(url, stream=True, timeout=30)
    response.raise_for_status()

    path = Path(save_path)
    path.parent.mkdir(parents=True, exist_ok=True)

    with open(path, "wb") as f:
        for chunk in response.iter_content(chunk_size=8192):
            f.write(chunk)

    print(f"✅ Downloaded: {path.name} ({path.stat().st_size / 1024:.1f} KB)")

download_file(
    url="https://example.com/reports/monthly_report.pdf",
    save_path="downloads/monthly_report.pdf"
)

from playwright.sync_api import sync_playwright

with sync_playwright() as p:
    browser = p.chromium.launch()
    context = browser.new_context(accept_downloads=True)
    page    = context.new_page()

    page.goto("https://example.com/dashboard")

    # Click the download button and capture the file
    with page.expect_download() as download_info:
        page.click("button#export-csv")

    download = download_info.value
    download.save_as(f"downloads/{download.suggested_filename}")
    print(f"✅ Saved: {download.suggested_filename}")

    browser.close()

# ── Playwright: waits are built in ──────────────────────────────
# page.fill() / page.click() automatically wait for the element
# You can also wait explicitly:

page.wait_for_selector(".results-list")          # wait until visible
page.wait_for_selector(".spinner", state="hidden")  # wait until loading spinner disappears
page.wait_for_load_state("networkidle")           # wait until no network requests
page.wait_for_url("**/success**")               # wait until URL matches pattern

# ── Selenium: use WebDriverWait ──────────────────────────────────
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

wait = WebDriverWait(driver, timeout=15)

# Wait until element appears in DOM
element = wait.until(EC.presence_of_element_located((By.ID, "result-table")))

# Wait until element is visible and clickable
btn = wait.until(EC.element_to_be_clickable((By.CSS_SELECTOR, ".submit-btn")))
btn.click()

# Wait until text appears on page
wait.until(EC.text_to_be_present_in_element((By.TAG_NAME, "body"), "Upload complete"))

# Scroll to bottom to trigger lazy-loaded content
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")

Tutorial

Selenium Python Tutorial 2026

Deep dive into Selenium: browser testing, web scraping, and automated interactions.

Tutorial

How to Automate File Management with Python

Organize files automatically by type or date, bulk rename, find duplicates.

Tutorial

How to Automate Excel Tasks with Python

Read, write, format, and auto-generate Excel reports using openpyxl and pandas.