Mobile versionhttps://www.oddsportal.com/pl/football/ ... jQH/#1X2;2
Mit Playwright habe ich eine Antwortabfangfunktion hinzugefügt, die aber nichts zurückgibt. Wir möchten die Daten zurückgeben, bevor sie auf der Seite gerendert werden
zB. Für die Anfrage https://www.oddsportal.com/pl/match-eve ... IN&lang=pl
gibt die API diese Daten zurück WDlrWUpLRU1mYXBwWXZkQnR5NklhZ0s4MDU5VzkxVkQ5cXpYejF2QzJxdkNNeUQrbEd1b1p1cUhRZnQrZHo1NVdaT3UwT
Ich muss es entschlüsseln, als ich im Browser-Devtools-Netzwerk-Tab eingecheckt habe
Code: Select all
[
{
"scraped_date": "2025-12-20T07:56:51.594544+00:00",
"match_date": "2025-12-20T12:00:00+00:00",
"home_team": "Fortuna D\u00fcsseldorf",
"away_team": "Furth",
"league_name": "2. Bundesliga",
"home_score": "",
"away_score": "",
"partial_results": "",
"details": {
"over_under_market": [
{
"submarket_name": "Over/Under +0.5",
"period": "FullTime",
"market_type": "Over/Under",
"extraction_mode": "passive",
"odds_over": "1.02",
"odds_under": "14.02"
},
{
"submarket_name": "Over/Under +0.75",
"period": "FullTime",
"market_type": "Over/Under",
"extraction_mode": "passive",
"odds_over": "1.03",
"odds_under": "10.50"
},
{
"submarket_name": "Over/Under +1",
"period": "FullTime",
"market_type": "Over/Under",
"extraction_mode": "passive",
"odds_over": "1.03",
"odds_under": "11.29"
},
{
"submarket_name": "Over/Under +1.25",
"period": "FullTime",
"market_type": "Over/Under",
"extraction_mode": "passive",
"odds_over": "1.09",
"odds_under": "5.89"
},
{
"submarket_name": "Over/Under +1.5",
"period": "FullTime",
"market_type": "Over/Under",
"extraction_mode": "passive",
"odds_over": "1.18",
"odds_under": "4.37"
},
{
"submarket_name": "Over/Under +1.75",
"period": "FullTime",
"market_type": "Over/Under",
"extraction_mode": "passive",
"odds_over": "1.21",
"odds_under": "3.93"
},
{
"submarket_name": "Over/Under +2",
"period": "FullTime",
"market_type": "Over/Under",
"extraction_mode": "passive",
"odds_over": "1.27",
"odds_under": "3.47"
},
{
"submarket_name": "Over/Under +2.25",
"period": "FullTime",
"market_type": "Over/Under",
"extraction_mode": "passive",
"odds_over": "1.44",
"odds_under": "2.61"
},
{
"submarket_name": "Over/Under +2.5",
"period": "FullTime",
"market_type": "Over/Under",
"extraction_mode": "passive",
"odds_over": "1.63",
"odds_under": "2.23"
},
{
"submarket_name": "Over/Under +2.75",
"period": "FullTime",
"market_type": "Over/Under",
"extraction_mode": "passive",
"odds_over": "1.77",
"odds_under": "1.98"
},
{
"submarket_name": "Over/Under +3",
"period": "FullTime",
"market_type": "Over/Under",
"extraction_mode": "passive",
"odds_over": "2.00",
"odds_under": "1.76"
},
{
"submarket_name": "Over/Under +3.25",
"period": "FullTime",
"market_type": "Over/Under",
"extraction_mode": "passive",
"odds_over": "2.24",
"odds_under": "1.59"
},
{
"submarket_name": "Over/Under +3.5",
"period": "FullTime",
"market_type": "Over/Under",
"extraction_mode": "passive",
"odds_over": "2.50",
"odds_under": "1.48"
},
{
"submarket_name": "Over/Under +3.75",
"period": "FullTime",
"market_type": "Over/Under",
"extraction_mode": "passive",
"odds_over": "2.90",
"odds_under": "1.35"
},
{
"submarket_name": "Over/Under +4",
"period": "FullTime",
"market_type": "Over/Under",
"extraction_mode": "passive",
"odds_over": "3.72",
"odds_under": "1.22"
},
{
"submarket_name": "Over/Under +4.25",
"period": "FullTime",
"market_type": "Over/Under",
"extraction_mode": "passive",
"odds_over": "4.06",
"odds_under": "1.19"
},
{
"submarket_name": "Over/Under +4.5",
"period": "FullTime",
"market_type": "Over/Under",
"extraction_mode": "passive",
"odds_over": "4.42",
"odds_under": "1.17"
},
{
"submarket_name": "Over/Under +4.75",
"period": "FullTime",
"market_type": "Over/Under",
"extraction_mode": "passive",
"odds_over": "5.42",
"odds_under": "1.11"
},
{
"submarket_name": "Over/Under +5",
"period": "FullTime",
"market_type": "Over/Under",
"extraction_mode": "passive",
"odds_over": "7.41",
"odds_under": "1.05"
},
{
"submarket_name": "Over/Under +5.25",
"period": "FullTime",
"market_type": "Over/Under",
"extraction_mode": "passive",
"odds_over": "7.50",
"odds_under": "1.07"
},
{
"submarket_name": "Over/Under +5.5",
"period": "FullTime",
"market_type": "Over/Under",
"extraction_mode": "passive",
"odds_over": "8.28",
"odds_under": "1.04"
},
{
"submarket_name": "Over/Under +5.75",
"period": "FullTime",
"market_type": "Over/Under",
"extraction_mode": "passive",
"odds_over": "9.25",
"odds_under": "1.04"
},
{
"submarket_name": "Over/Under +6",
"period": "FullTime",
"market_type": "Over/Under",
"extraction_mode": "passive",
"odds_over": "16.21",
"odds_under": "1.01"
},
{
"submarket_name": "Over/Under +6.5",
"period": "FullTime",
"market_type": "Over/Under",
"extraction_mode": "passive",
"odds_over": "16.60",
"odds_under": "1.02"
},
{
"submarket_name": "Over/Under +7",
"period": "FullTime",
"market_type": "Over/Under",
"extraction_mode": "passive",
"odds_over": "91.08",
"odds_under": "1.00"
},
{
"submarket_name": "Over/Under +7.5",
"period": "FullTime",
"market_type": "Over/Under",
"extraction_mode": "passive",
"odds_over": "23.80",
"odds_under": "1.01"
}
],
"asian_handicap_market": [
{
"submarket_name": "Asian Handicap -4.5",
"period": "FullTime",
"market_type": "Asian Handicap",
"extraction_mode": "passive",
"team1_handicap": "23.72",
"team2_handicap": "1.00"
},
{
"submarket_name": "Asian Handicap -3.5",
"period": "FullTime",
"market_type": "Asian Handicap",
"extraction_mode": "passive",
"team1_handicap": "12.28",
"team2_handicap": "1.01"
},
{
"submarket_name": "Asian Handicap -3.25",
"period": "FullTime",
"market_type": "Asian Handicap",
"extraction_mode": "passive",
"team1_handicap": "12.00",
"team2_handicap": "1.02"
},
{
"submarket_name": "Asian Handicap -3",
"period": "FullTime",
"market_type": "Asian Handicap",
"extraction_mode": "passive",
"team1_handicap": "11.70",
"team2_handicap": "1.02"
},
{
"submarket_name": "Asian Handicap -2.75",
"period": "FullTime",
"market_type": "Asian Handicap",
"extraction_mode": "passive",
"team1_handicap": "7.69",
"team2_handicap": "1.07"
},
{
"submarket_name": "Asian Handicap -2.5",
"period": "FullTime",
"market_type": "Asian Handicap",
"extraction_mode": "passive",
"team1_handicap": "6.03",
"team2_handicap": "1.09"
},
{
"submarket_name": "Asian Handicap -2.25",
"period": "FullTime",
"market_type": "Asian Handicap",
"extraction_mode": "passive",
"team1_handicap": "5.63",
"team2_handicap": "1.10"
},
{
"submarket_name": "Asian Handicap -2",
"period": "FullTime",
"market_type": "Asian Handicap",
"extraction_mode": "passive",
"team1_handicap": "5.24",
"team2_handicap": "1.12"
},
{
"submarket_name": "Asian Handicap -1.75",
"period": "FullTime",
"market_type": "Asian Handicap",
"extraction_mode": "passive",
"team1_handicap": "3.83",
"team2_handicap": "1.21"
},
{
"submarket_name": "Asian Handicap -1.5",
"period": "FullTime",
"market_type": "Asian Handicap",
"extraction_mode": "passive",
"team1_handicap": "3.13",
"team2_handicap": "1.33"
},
{
"submarket_name": "Asian Handicap -1.25",
"period": "FullTime",
"market_type": "Asian Handicap",
"extraction_mode": "passive",
"team1_handicap": "2.81",
"team2_handicap": "1.39"
},
{
"submarket_name": "Asian Handicap -1",
"period": "FullTime",
"market_type": "Asian Handicap",
"extraction_mode": "passive",
"team1_handicap": "2.51",
"team2_handicap": "1.50"
},
{
"submarket_name": "Asian Handicap -0.75",
"period": "FullTime",
"market_type": "Asian Handicap",
"extraction_mode": "passive",
"team1_handicap": "2.06",
"team2_handicap": "1.70"
},
{
"submarket_name": "Asian Handicap -0.5",
"period": "FullTime",
"market_type": "Asian Handicap",
"extraction_mode": "passive",
"team1_handicap": "1.87",
"team2_handicap": "1.91"
},
{
"submarket_name": "Asian Handicap -0.25",
"period": "FullTime",
"market_type": "Asian Handicap",
"extraction_mode": "passive",
"team1_handicap": "1.62",
"team2_handicap": "2.20"
},
{
"submarket_name": "Asian Handicap 0",
"period": "FullTime",
"market_type": "Asian Handicap",
"extraction_mode": "passive",
"team1_handicap": "1.39",
"team2_handicap": "2.79"
},
{
"submarket_name": "Asian Handicap +0.25",
"period": "FullTime",
"market_type": "Asian Handicap",
"extraction_mode": "passive",
"team1_handicap": "1.31",
"team2_handicap": "3.20"
},
{
"submarket_name": "Asian Handicap +0.5",
"period": "FullTime",
"market_type": "Asian Handicap",
"extraction_mode": "passive",
"team1_handicap": "1.27",
"team2_handicap": "3.71"
},
{
"submarket_name": "Asian Handicap +0.75",
"period": "FullTime",
"market_type": "Asian Handicap",
"extraction_mode": "passive",
"team1_handicap": "1.15",
"team2_handicap": "4.58"
},
{
"submarket_name": "Asian Handicap +1",
"period": "FullTime",
"market_type": "Asian Handicap",
"extraction_mode": "passive",
"team1_handicap": "1.07",
"team2_handicap": "6.78"
},
{
"submarket_name": "Asian Handicap +1.25",
"period": "FullTime",
"market_type": "Asian Handicap",
"extraction_mode": "passive",
"team1_handicap": "1.05",
"team2_handicap": "7.14"
},
{
"submarket_name": "Asian Handicap +1.5",
"period": "FullTime",
"market_type": "Asian Handicap",
"extraction_mode": "passive",
"team1_handicap": "1.04",
"team2_handicap": "7.58"
},
{
"submarket_name": "Asian Handicap +1.75",
"period": "FullTime",
"market_type": "Asian Handicap",
"extraction_mode": "passive",
"team1_handicap": "1.05",
"team2_handicap": "8.75"
},
{
"submarket_name": "Asian Handicap +2",
"period": "FullTime",
"market_type": "Asian Handicap",
"extraction_mode": "passive",
"team1_handicap": "1.01",
"team2_handicap": "13.42"
},
{
"submarket_name": "Asian Handicap +3",
"period": "FullTime",
"market_type": "Asian Handicap",
"extraction_mode": "passive",
"team1_handicap": "1.00",
"team2_handicap": "39.23"
},
{
"submarket_name": "Asian Handicap +4",
"period": "FullTime",
"market_type": "Asian Handicap",
"extraction_mode": "passive",
"team1_handicap": "-",
"team2_handicap": "-"
},
{
"submarket_name": "Asian Handicap +4.5",
"period": "FullTime",
"market_type": "Asian Handicap",
"extraction_mode": "passive",
"team1_handicap": "-",
"team2_handicap": "-"
},
{
"submarket_name": "Asian Handicap +5",
"period": "FullTime",
"market_type": "Asian Handicap",
"extraction_mode": "passive",
"team1_handicap": "1.00",
"team2_handicap": "100.00"
}
]
},
"_network_payloads": []
}
]
Code: Select all
import asyncio
import json
import logging
import re
from datetime import UTC, datetime
from typing import Any
from bs4 import BeautifulSoup
from playwright.async_api import Page, TimeoutError
from src.utils.constants import ODDSPORTAL_BASE_URL
from src.utils.odds_format_enum import OddsFormat
from src.utils.utils import clean_html_text
class BaseScraper:
"""
Base class for scraping match data from OddsPortal
using DOM parsing + network interception.
"""
def __init__(
self,
playwright_manager,
browser_helper,
market_extractor,
preview_submarkets_only: bool = False,
):
self.logger = logging.getLogger(self.__class__.__name__)
self.playwright_manager = playwright_manager
self.browser_helper = browser_helper
self.market_extractor = market_extractor
self.preview_submarkets_only = preview_submarkets_only
# ---------------------------------------------------------------------
# NETWORK INTERCEPTION (WHAT YOUR CLIENT WANTS)
# ---------------------------------------------------------------------
async def _attach_response_interceptor(
self,
page: Page,
captured: list[dict[str, Any]],
):
async def handle_response(response):
url = response.url.lower()
if any(k in url for k in ("feed", "ajax", "match", "odds")):
try:
content_type = response.headers.get("content-type", "")
if content_type.startswith("application/json"):
data = await response.json()
captured.append({
"url": response.url,
"data": data,
"captured_at": datetime.now(UTC).isoformat(),
})
self.logger.debug(f"Captured JSON → {response.url}")
except Exception:
# silently ignore non-json / encrypted payloads
pass
self.logger.info(f"Trying to get response...")
page.on("response", handle_response)
# ---------------------------------------------------------------------
async def set_odds_format(self, page: Page, odds_format: OddsFormat = OddsFormat.DECIMAL_ODDS):
try:
button_selector = "div.group > button.gap-2"
await page.wait_for_selector(button_selector, timeout=8000)
dropdown_button = await page.query_selector(button_selector)
current_format = await dropdown_button.inner_text()
if current_format == odds_format.value:
return
await dropdown_button.click()
await page.wait_for_timeout(1000)
options = await page.query_selector_all("div.dropdown-content a")
for option in options:
if odds_format.value.lower() in (await option.inner_text()).lower():
await option.click()
await page.wait_for_timeout(1000)
return
except Exception as e:
self.logger.warning(f"Failed to set odds format: {e}")
# ---------------------------------------------------------------------
async def extract_match_links(self, page: Page) -> list[str]:
html = await page.content()
soup = BeautifulSoup(html, "html.parser")
links = set()
for row in soup.find_all(class_=re.compile("^eventRow")):
a = row.find("a", href=True)
if a and len(a["href"].strip("/").split("/")) > 3:
links.add(f"{ODDSPORTAL_BASE_URL}{a['href']}")
return list(links)
# ---------------------------------------------------------------------
async def extract_match_odds(
self,
sport: str,
match_links: list[str],
markets: list[str] | None = None,
scrape_odds_history: bool = False,
target_bookmaker: str | None = None,
concurrent_scraping_task: int = 3,
preview_submarkets_only: bool = False,
) -> list[dict[str, Any]]:
semaphore = asyncio.Semaphore(concurrent_scraping_task)
results = []
async def worker(link: str):
async with semaphore:
page = await self.playwright_manager.context.new_page()
captured_responses: list[dict] = []
try:
await self._attach_response_interceptor(page, captured_responses)
await page.goto(link, wait_until="networkidle", timeout=30000)
await page.wait_for_timeout(1500)
match_details = await self._extract_match_details_event_header(page)
if not match_details:
return None
if markets:
market_data = await self.market_extractor.scrape_markets(
page=page,
sport=sport,
markets=markets,
period="FullTime",
scrape_odds_history=scrape_odds_history,
target_bookmaker=target_bookmaker,
preview_submarkets_only=preview_submarkets_only,
)
match_details["details"].update(market_data or {})
# attach raw intercepted JSON
match_details["_network_payloads"] = captured_responses
return match_details
finally:
await page.close()
tasks = [worker(link) for link in match_links]
for result in await asyncio.gather(*tasks):
if result:
results.append(result)
return results
# ---------------------------------------------------------------------
async def _extract_match_details_event_header(self, page: Page) -> dict[str, Any] | None:
try:
await page.wait_for_selector("#react-event-header", timeout=10000)
soup = BeautifulSoup(await page.content(), "html.parser")
header = soup.find("div", id="react-event-header")
if not header or not header.get("data"):
return None
data = json.loads(header["data"])
event_body = data.get("eventBody", {})
event_data = data.get("eventData", {})
match_date = (
datetime.fromtimestamp(event_body.get("startDate"), tz=UTC)
if event_body.get("startDate")
else None
)
return {
"scraped_date": datetime.now(UTC).isoformat(),
"match_date": match_date.isoformat() if match_date else None,
"home_team": event_data.get("home"),
"away_team": event_data.get("away"),
"league_name": event_data.get("tournamentName"),
"home_score": event_body.get("homeResult"),
"away_score": event_body.get("awayResult"),
"partial_results": clean_html_text(event_body.get("partialresult")),
"details": {},
}
except Exception as e:
self.logger.error(f"Header extraction failed: {e}")
return None
