Scraping de sites com autenticação: guia completo com exemplos de código

import requests
from bs4 import BeautifulSoup

LOGIN_URL = "https://supplier.example.com/login"
PRICES_URL = "https://supplier.example.com/account/prices"

session = requests.Session()
session.headers.update({
    "User-Agent": "Mozilla/5.0 (compatible; PriceMonitor/1.0)"
})

# 1. Carregamos a página de login e extraímos o token CSRF
login_page = session.get(LOGIN_URL, timeout=30)
soup = BeautifulSoup(login_page.text, "html.parser")
csrf_token = soup.select_one('input[name="csrf_token"]')["value"]

# 2. Enviamos o formulário de login
payload = {
    "username": "your_login",
    "password": "your_password",
    "csrf_token": csrf_token,
}
resp = session.post(LOGIN_URL, data=payload, timeout=30)
resp.raise_for_status()

if "Minha conta" not in resp.text:
    raise RuntimeError("Falha na autenticação: verifique as credenciais")

# 3. Sessão estabelecida: solicitamos a página privada de preços
prices_page = session.get(PRICES_URL, timeout=30)
soup = BeautifulSoup(prices_page.text, "html.parser")

for row in soup.select("table.prices tr"):
    cells = row.select("td")
    if len(cells) >= 2:
        name = cells[0].get_text(strip=True)
        price = cells[1].get_text(strip=True)
        print(f"{name}: {price}")

import requests

auth = requests.post(
    "https://supplier.example.com/api/auth/login",
    json={"login": "your_login", "password": "your_password"},
    timeout=30,
)
auth.raise_for_status()
token = auth.json()["access_token"]

headers = {"Authorization": f"Bearer {token}"}
data = requests.get(
    "https://supplier.example.com/api/prices",
    headers=headers,
    timeout=30,
).json()

for item in data["items"]:
    print(item["sku"], item["price"])

from playwright.sync_api import sync_playwright

with sync_playwright() as p:
    browser = p.chromium.launch(headless=True)
    context = browser.new_context()
    page = context.new_page()

    # Login
    page.goto("https://supplier.example.com/login")
    page.fill("input[name='username']", "your_login")
    page.fill("input[name='password']", "your_password")
    page.click("button[type='submit']")
    page.wait_for_url("**/account/**")

    # Salvamos a sessão para reutilizá-la mais tarde
    context.storage_state(path="auth_state.json")

    # Vamos aos preços e esperamos os dados carregarem
    page.goto("https://supplier.example.com/account/prices")
    page.wait_for_selector("table.prices")

    rows = page.query_selector_all("table.prices tr")
    for row in rows:
        cells = row.query_selector_all("td")
        if len(cells) >= 2:
            print(cells[0].inner_text(), "—", cells[1].inner_text())

    browser.close()

const axios = require("axios");
const { wrapper } = require("axios-cookiejar-support");
const { CookieJar } = require("tough-cookie");
const cheerio = require("cheerio");

const jar = new CookieJar();
const client = wrapper(axios.create({ jar, withCredentials: true }));

async function run() {
  // 1. Obtemos o token CSRF da página de login
  const loginPage = await client.get("https://supplier.example.com/login");
  const $ = cheerio.load(loginPage.data);
  const csrf = $('input[name="csrf_token"]').val();

  // 2. Fazemos login
  await client.post(
    "https://supplier.example.com/login",
    new URLSearchParams({
      username: "your_login",
      password: "your_password",
      csrf_token: csrf,
    }),
  );

  // 3. Solicitamos os preços
  const pricesPage = await client.get(
    "https://supplier.example.com/account/prices",
  );
  const $$ = cheerio.load(pricesPage.data);

  $$("table.prices tr").each((_, el) => {
    const cells = $$(el).find("td");
    if (cells.length >= 2) {
      const name = $$(cells[0]).text().trim();
      const price = $$(cells[1]).text().trim();
      console.log(`${name}: ${price}`);
    }
  });
}

run().catch(console.error);

const puppeteer = require("puppeteer");

(async () => {
  const browser = await puppeteer.launch({ headless: true });
  const page = await browser.newPage();

  await page.goto("https://supplier.example.com/login");
  await page.type("input[name='username']", "your_login");
  await page.type("input[name='password']", "your_password");
  await Promise.all([
    page.click("button[type='submit']"),
    page.waitForNavigation(),
  ]);

  await page.goto("https://supplier.example.com/account/prices");
  await page.waitForSelector("table.prices");

  const prices = await page.evaluate(() =>
    Array.from(document.querySelectorAll("table.prices tr"))
      .map((row) => {
        const td = row.querySelectorAll("td");
        return td.length >= 2
          ? { name: td[0].innerText.trim(), price: td[1].innerText.trim() }
          : null;
      })
      .filter(Boolean),
  );

  console.log(prices);
  await browser.close();
})();

<?php
$cookieFile = __DIR__ . "/cookies.txt";

function curlInit(string $cookieFile) {
    $ch = curl_init();
    curl_setopt_array($ch, [
        CURLOPT_RETURNTRANSFER => true,
        CURLOPT_FOLLOWLOCATION => true,
        CURLOPT_COOKIEJAR      => $cookieFile,
        CURLOPT_COOKIEFILE     => $cookieFile,
        CURLOPT_USERAGENT      => "Mozilla/5.0 (compatible; PriceMonitor/1.0)",
    ]);
    return $ch;
}

// 1. Carregamos a página de login e extraímos o token CSRF
$ch = curlInit($cookieFile);
curl_setopt($ch, CURLOPT_URL, "https://supplier.example.com/login");
$html = curl_exec($ch);

preg_match('/name="csrf_token"\s+value="([^"]+)"/', $html, $m);
$csrf = $m[1] ?? "";

// 2. Enviamos o formulário de login
curl_setopt($ch, CURLOPT_URL, "https://supplier.example.com/login");
curl_setopt($ch, CURLOPT_POST, true);
curl_setopt($ch, CURLOPT_POSTFIELDS, http_build_query([
    "username"   => "your_login",
    "password"   => "your_password",
    "csrf_token" => $csrf,
]));
curl_exec($ch);

// 3. Solicitamos os preços
curl_setopt($ch, CURLOPT_URL, "https://supplier.example.com/account/prices");
curl_setopt($ch, CURLOPT_POST, false);
$pricesHtml = curl_exec($ch);
curl_close($ch);

// Parsing do HTML
$dom = new DOMDocument();
@$dom->loadHTML($pricesHtml);
$xpath = new DOMXPath($dom);
foreach ($xpath->query("//table[@class='prices']//tr") as $row) {
    $cells = $row->getElementsByTagName("td");
    if ($cells->length >= 2) {
        echo trim($cells->item(0)->textContent) . ": "
           . trim($cells->item(1)->textContent) . PHP_EOL;
    }
}

package main

import (
    "fmt"
    "net/http"
    "net/http/cookiejar"
    "net/url"
    "strings"

    "github.com/PuerkitoBio/goquery"
)

func main() {
    jar, _ := cookiejar.New(nil)
    client := &http.Client{Jar: jar}

    // 1. Obtemos o token CSRF
    resp, _ := client.Get("https://supplier.example.com/login")
    doc, _ := goquery.NewDocumentFromReader(resp.Body)
    resp.Body.Close()
    csrf, _ := doc.Find(`input[name="csrf_token"]`).Attr("value")

    // 2. Fazemos login
    form := url.Values{
        "username":   {"your_login"},
        "password":   {"your_password"},
        "csrf_token": {csrf},
    }
    client.Post(
        "https://supplier.example.com/login",
        "application/x-www-form-urlencoded",
        strings.NewReader(form.Encode()),
    )

    // 3. Extraímos os preços
    pricesResp, _ := client.Get("https://supplier.example.com/account/prices")
    pricesDoc, _ := goquery.NewDocumentFromReader(pricesResp.Body)
    pricesResp.Body.Close()

    pricesDoc.Find("table.prices tr").Each(func(_ int, s *goquery.Selection) {
        cells := s.Find("td")
        if cells.Length() >= 2 {
            name := strings.TrimSpace(cells.Eq(0).Text())
            price := strings.TrimSpace(cells.Eq(1).Text())
            fmt.Printf("%s: %s\n", name, price)
        }
    })
}