From fe20a4cef8f4414d21dda9c4a88eabd25b9de637 Mon Sep 17 00:00:00 2001 From: claiireyu Date: Wed, 22 Apr 2026 14:58:04 -0400 Subject: [PATCH 1/4] Added recap link scraping for sports that do not have box scores - Added optional fields for game recap links, article titles, and published dates in the GameType and Game models. - Updated CreateGame mutation to accept new recap fields. - Implemented a new scraping function to extract recap headlines and published times from Cornell Sidearm story pages. --- src/models/game.py | 15 ++++++ src/mutations/create_game.py | 13 ++++- src/scrapers/game_details_scrape.py | 41 +++++++++++++- src/scrapers/games_scraper.py | 83 +++++++++++++++++++++-------- src/types.py | 13 ++++- src/utils/constants.py | 31 +++++++++++ 6 files changed, 169 insertions(+), 27 deletions(-) diff --git a/src/models/game.py b/src/models/game.py index 73a7968..e2e79e5 100644 --- a/src/models/game.py +++ b/src/models/game.py @@ -18,6 +18,9 @@ class Game: - `box_score` The scoring summary of the game (optional) - `score_breakdown` The scoring breakdown of the game (optional) - 'ticket_link' The ticket link for the game (optional) + - 'recap_link' The recap/details link for the game (optional) + - 'recap_article_title' Title from the recap/story page when scraped (optional) + - 'recap_published_at' Published date/time string from the recap page (optional) """ def __init__( @@ -37,6 +40,9 @@ def __init__( team=None, utc_date=None, ticket_link=None, + recap_link=None, + recap_article_title=None, + recap_published_at=None, ): self.id = id if id else str(ObjectId()) self.city = city @@ -53,6 +59,9 @@ def __init__( self.team = team self.utc_date = utc_date self.ticket_link = ticket_link + self.recap_link = recap_link + self.recap_article_title = recap_article_title + self.recap_published_at = recap_published_at def to_dict(self): """ @@ -74,6 +83,9 @@ def to_dict(self): "team": self.team, "utc_date": self.utc_date, "ticket_link": self.ticket_link, + "recap_link": self.recap_link, + "recap_article_title": self.recap_article_title, + "recap_published_at": self.recap_published_at, } @staticmethod @@ -97,4 +109,7 @@ def from_dict(data) -> None: team=data.get("team"), utc_date=data.get("utc_date"), ticket_link=data.get("ticket_link"), + recap_link=data.get("recap_link"), + recap_article_title=data.get("recap_article_title"), + recap_published_at=data.get("recap_published_at"), ) diff --git a/src/mutations/create_game.py b/src/mutations/create_game.py index 3a52345..28d2ce6 100644 --- a/src/mutations/create_game.py +++ b/src/mutations/create_game.py @@ -18,6 +18,9 @@ class Arguments: score_breakdown = String(required=False) utc_date = String(required=False) ticket_link = String(required=False) + recap_link = String(required=False) + recap_article_title = String(required=False) + recap_published_at = String(required=False) game = Field(lambda: GameType) @@ -36,7 +39,10 @@ def mutate( box_score=None, score_breakdown=None, utc_date=None, - ticket_link=None + ticket_link=None, + recap_link=None, + recap_article_title=None, + recap_published_at=None, ): game_data = { "city": city, @@ -51,7 +57,10 @@ def mutate( "box_score": box_score, "score_breakdown": score_breakdown, "utc_date": utc_date, - "ticket_link": ticket_link + "ticket_link": ticket_link, + "recap_link": recap_link, + "recap_article_title": recap_article_title, + "recap_published_at": recap_published_at, } new_game = GameService.create_game(game_data) return CreateGame(game=new_game) \ No newline at end of file diff --git a/src/scrapers/game_details_scrape.py b/src/scrapers/game_details_scrape.py index 5f2f3b1..50eae76 100644 --- a/src/scrapers/game_details_scrape.py +++ b/src/scrapers/game_details_scrape.py @@ -22,9 +22,46 @@ def clean_name(name): return cleaned def fetch_page(url): - response = requests.get(url) + response = requests.get(url, headers=HTTP_REQUEST_HEADERS, timeout=20) return BeautifulSoup(response.text, 'html.parser') + +def scrape_sidearm_story_recap(url): + """ + Extract headline and published time from a Cornell Sidearm story/recap page + """ + if not url: + return {} + try: + response = requests.get(url, headers=HTTP_REQUEST_HEADERS, timeout=20) + if response.status_code != 200: + return {} + soup = BeautifulSoup(response.text, "html.parser") + except Exception: + return {} + headline = soup.select_one(SIDEARM_STORY_HEADLINE) + time_el = soup.select_one(SIDEARM_STORY_PUBLISHED_TIME) + title = headline.get_text(strip=True) if headline else None + if not title: + og = soup.find("meta", property="og:title") + if og and og.get("content"): + title = og["content"].strip() + published_at = None + if time_el: + published_at = time_el.get_text(strip=True) + if not published_at and time_el.get("datetime"): + published_at = time_el["datetime"].strip() + if not published_at: + pmeta = soup.find("meta", property="article:published_time") + if pmeta and pmeta.get("content"): + published_at = pmeta["content"].strip() + out = {} + if title: + out["recap_article_title"] = title + if published_at: + out["recap_published_at"] = published_at + return out + def extract_teams_and_scores(box_score_section, sport): score_table = box_score_section.find(TAG_TABLE, class_=CLASS_SIDEARM_TABLE) team_names = [] @@ -229,6 +266,7 @@ def baseball_summary(box_score_section): summary = [{"message": "No scoring events in this game."}] return summary + # def basketball_summary(box_score_section): # summary = [] # scoring_section = box_score_section.find(TAG_SECTION, {ATTR_ARIA_LABEL: LABEL_SCORING_SUMMARY}) @@ -273,6 +311,7 @@ def scrape_game(url, sport): 'lacrosse': (lambda: extract_teams_and_scores(box_score_section, 'lacrosse'), lacrosse_summary), 'baseball': (lambda: extract_teams_and_scores(box_score_section, 'baseball'), baseball_summary), 'basketball': (lambda: extract_teams_and_scores(box_score_section, 'basketball'), lambda _: []), + } extract_teams_func, summary_func = sport_parsers.get(sport, (None, None)) diff --git a/src/scrapers/games_scraper.py b/src/scrapers/games_scraper.py index 818760c..ddd393c 100644 --- a/src/scrapers/games_scraper.py +++ b/src/scrapers/games_scraper.py @@ -3,7 +3,7 @@ from src.services import GameService, TeamService from src.utils.convert_to_utc import convert_to_utc from src.utils.constants import * -from src.scrapers.game_details_scrape import scrape_game +from src.scrapers.game_details_scrape import scrape_game, scrape_sidearm_story_recap from src.utils.helpers import get_dominant_color, normalize_game_data, is_tournament_placeholder_team, is_cornell_loss import base64 import re @@ -40,6 +40,38 @@ def infer_game_year(date_text, season_years): return second_year return first_year + +def to_absolute_url(link): + """Convert relative Cornell links like /news/... to absolute URLs.""" + if not link: + return None + if link.startswith("http://") or link.startswith("https://"): + return link + return f"{BASE_URL.rstrip('/')}/{link.lstrip('/')}" + + +def parse_game_links(game_item): + """ + Parse link info for a schedule item. + Returns (recap_link, ticket_link). + """ + recap_link = None + box_score_tag = game_item.select_one(BOX_SCORE_TAG) + if box_score_tag: + box_score_link = box_score_tag.get("href") + recap_link = to_absolute_url(box_score_link) + + # Many sports expose recap links here when there is no separate box score link. + if not recap_link: + recap_tag = game_item.select_one(RECAP_TAG) + if recap_tag: + recap_link = to_absolute_url(recap_tag.get("href")) + + ticket_link_tag = game_item.select_one(GAME_TICKET_LINK) + ticket_link = to_absolute_url(ticket_link_tag["href"]) if ticket_link_tag else None + + return recap_link, ticket_link + def fetch_game_schedule(): """ Scrape the game schedule from the given URLs in parallel using threads. @@ -71,7 +103,7 @@ def parse_schedule_page(url, sport, gender): sport (str): The sport of the games. gender (str): The gender of the games. """ - response = requests.get(url) + response = requests.get(url, headers=HTTP_REQUEST_HEADERS, timeout=30) soup = BeautifulSoup(response.content, "html.parser") page_title = soup.title.text.strip() if soup.title else "" @@ -129,11 +161,15 @@ def parse_schedule_page(url, sport, gender): else: game_data["result"] = None - box_score_tag = game_item.select_one(BOX_SCORE_TAG) - if box_score_tag: - box_score_link = box_score_tag["href"] - game_details = scrape_game(f"{BASE_URL}{box_score_link}", sport.lower()) - if game_details.get('error') == 'Sport parser not found': + recap_link, ticket_link = parse_game_links(game_item) + + # These sports use news/recap pages instead of parsable box-score HTML. + if sport in SPORTS_WITH_SIDEARM_STORY_RECAP or not recap_link: + game_data["box_score"] = None + game_data["score_breakdown"] = None + else: + game_details = scrape_game(recap_link, sport.lower()) + if game_details.get("error") == "Sport parser not found": game_data["box_score"] = None game_data["score_breakdown"] = None else: @@ -144,23 +180,17 @@ def parse_schedule_page(url, sport, gender): location_data = game_data["location"].split("\n") if game_data["location"] else [""] geo_location = location_data[0] is_home_game = "Ithaca" in geo_location - + if is_home_game and game_data["box_score"]: for event in game_data["box_score"]: if "cor_score" in event and "opp_score" in event: event["cor_score"], event["opp_score"] = event["opp_score"], event["cor_score"] - else: - game_data["box_score"] = None - game_data["score_breakdown"] = None - - ticket_link_tag = game_item.select_one(GAME_TICKET_LINK) - ticket_link = ( - ticket_link_tag["href"] if ticket_link_tag else None - ) - game_data["ticket_link"] = ( - ticket_link if ticket_link else None - ) + if sport in SPORTS_WITH_SIDEARM_STORY_RECAP and recap_link: + game_data.update(scrape_sidearm_story_recap(recap_link)) + + game_data["ticket_link"] = ticket_link + game_data["recap_link"] = recap_link process_game_data(game_data) @@ -173,7 +203,8 @@ def process_game_data(game_data): """ game_data = normalize_game_data(game_data) - location_data = game_data["location"].split("\n") + location_raw = game_data.get("location") or "" + location_data = location_raw.split("\n") if location_raw else [""] geo_location = location_data[0] if (",") not in geo_location: city = geo_location @@ -278,8 +309,13 @@ def process_game_data(game_data): "city": city, "location": location, "state": state, - "ticket_link": game_data["ticket_link"] + "ticket_link": game_data["ticket_link"], + "recap_link": game_data.get("recap_link"), } + if "recap_article_title" in game_data: + updates["recap_article_title"] = game_data["recap_article_title"] + if "recap_published_at" in game_data: + updates["recap_published_at"] = game_data["recap_published_at"] current_team = TeamService.get_team_by_id(curr_game.opponent_id) if current_team and is_tournament_placeholder_team(current_team.name): @@ -304,7 +340,10 @@ def process_game_data(game_data): "box_score": game_data["box_score"], "score_breakdown": game_data["score_breakdown"], "utc_date": utc_date_str, - "ticket_link": game_data["ticket_link"] + "ticket_link": game_data["ticket_link"], + "recap_link": game_data.get("recap_link"), + "recap_article_title": game_data.get("recap_article_title"), + "recap_published_at": game_data.get("recap_published_at"), } GameService.create_game(game_data) \ No newline at end of file diff --git a/src/types.py b/src/types.py index 7eb8fbe..743a286 100644 --- a/src/types.py +++ b/src/types.py @@ -1,5 +1,4 @@ from graphene import ObjectType, Field, String, List, Int -from datetime import datetime class TeamType(ObjectType): """ @@ -89,6 +88,9 @@ class GameType(ObjectType): - `box_score`: The box score of the game. - `score_breakdown`: The score breakdown of the game. - `ticket_link`: The ticket link of the game. (optional) + - `recap_link`: The recap/details link of the game. (optional) + - `recap_article_title`: Headline from the recap story page when scraped (optional) + - `recap_published_at`: Published date/time from the recap story page (optional) """ id = String(required=False) @@ -106,8 +108,11 @@ class GameType(ObjectType): team = Field(TeamType, required=False) utc_date = String(required=False) ticket_link = String(required=False) + recap_link = String(required=False) + recap_article_title = String(required=False) + recap_published_at = String(required=False) def __init__( - self, id, city, date, gender, location, opponent_id, result, sport, state, time, box_score=None, score_breakdown=None, utc_date=None, ticket_link=None + self, id, city, date, gender, location, opponent_id, result, sport, state, time, box_score=None, score_breakdown=None, utc_date=None, ticket_link=None, recap_link=None, recap_article_title=None, recap_published_at=None ): self.id = id self.city = city @@ -123,6 +128,10 @@ def __init__( self.score_breakdown = score_breakdown self.utc_date = utc_date self.ticket_link = ticket_link + self.recap_link = recap_link + self.recap_article_title = recap_article_title + self.recap_published_at = recap_published_at + @staticmethod def team_to_team_type(team_obj): if team_obj is None: diff --git a/src/utils/constants.py b/src/utils/constants.py index 38c2ae7..ab6ea6c 100644 --- a/src/utils/constants.py +++ b/src/utils/constants.py @@ -9,6 +9,30 @@ # Base URL BASE_URL = "https://cornellbigred.com" +# cornellbigred.com often returns 404 or empty HTML for Python's default requests User-Agent. +HTTP_REQUEST_HEADERS = { + "User-Agent": ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" + ), + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", +} + +# Schedule rows for these sports link to Sidearm story pages (no box score); scrape recap headline/date the same way. +SPORTS_WITH_SIDEARM_STORY_RECAP = frozenset( + { + "Swimming & Diving", + "Track & Field", + "Wrestling", + "Golf", + "Polo", + "Fencing", + "Equestrian", + "Gymnastics" + } +) + # The tag for each game GAME_TAG = ".sidearm-schedule-game" @@ -45,9 +69,16 @@ # The tag for the box score BOX_SCORE_TAG = ".sidearm-schedule-game-links-boxscore a" +# The tag for recap/details links +RECAP_TAG = ".sidearm-schedule-game-links-recap a" + # The tag for the game ticket link GAME_TICKET_LINK = ".sidearm-schedule-game-links-tickets a" +# Sidearm full story recap article page +SIDEARM_STORY_HEADLINE = "h1.sidearm-story-template-headline" +SIDEARM_STORY_PUBLISHED_TIME = ".sidearm-story-template-date time" + # HTML Tags TAG_TABLE = 'table' TAG_SECTION = 'section' From 947cc39a76c91f775e933bc316c9253bee00f90e Mon Sep 17 00:00:00 2001 From: claiireyu Date: Wed, 2 Sep 2026 22:41:17 -0400 Subject: [PATCH 2/4] Fixed scrapers for box score and recap link sports --- src/models/game.py | 5 ++ src/mutations/create_game.py | 5 +- src/scrapers/game_details_scrape.py | 16 ++++- src/scrapers/games_scraper.py | 106 ++++++++++++++++------------ src/types.py | 25 ++++++- src/utils/constants.py | 66 ++++++++++------- 6 files changed, 145 insertions(+), 78 deletions(-) diff --git a/src/models/game.py b/src/models/game.py index e2e79e5..68d400d 100644 --- a/src/models/game.py +++ b/src/models/game.py @@ -21,6 +21,7 @@ class Game: - 'recap_link' The recap/details link for the game (optional) - 'recap_article_title' Title from the recap/story page when scraped (optional) - 'recap_published_at' Published date/time string from the recap page (optional) + - 'recap_article_image' Primary image URL from the recap page (optional) """ def __init__( @@ -43,6 +44,7 @@ def __init__( recap_link=None, recap_article_title=None, recap_published_at=None, + recap_article_image=None, ): self.id = id if id else str(ObjectId()) self.city = city @@ -62,6 +64,7 @@ def __init__( self.recap_link = recap_link self.recap_article_title = recap_article_title self.recap_published_at = recap_published_at + self.recap_article_image = recap_article_image def to_dict(self): """ @@ -86,6 +89,7 @@ def to_dict(self): "recap_link": self.recap_link, "recap_article_title": self.recap_article_title, "recap_published_at": self.recap_published_at, + "recap_article_image": self.recap_article_image, } @staticmethod @@ -112,4 +116,5 @@ def from_dict(data) -> None: recap_link=data.get("recap_link"), recap_article_title=data.get("recap_article_title"), recap_published_at=data.get("recap_published_at"), + recap_article_image=data.get("recap_article_image"), ) diff --git a/src/mutations/create_game.py b/src/mutations/create_game.py index 28d2ce6..73a5354 100644 --- a/src/mutations/create_game.py +++ b/src/mutations/create_game.py @@ -21,6 +21,7 @@ class Arguments: recap_link = String(required=False) recap_article_title = String(required=False) recap_published_at = String(required=False) + recap_article_image = String(required=False) game = Field(lambda: GameType) @@ -43,6 +44,7 @@ def mutate( recap_link=None, recap_article_title=None, recap_published_at=None, + recap_article_image=None, ): game_data = { "city": city, @@ -61,6 +63,7 @@ def mutate( "recap_link": recap_link, "recap_article_title": recap_article_title, "recap_published_at": recap_published_at, + "recap_article_image": recap_article_image, } new_game = GameService.create_game(game_data) - return CreateGame(game=new_game) \ No newline at end of file + return CreateGame(game=new_game) diff --git a/src/scrapers/game_details_scrape.py b/src/scrapers/game_details_scrape.py index 50eae76..efa477d 100644 --- a/src/scrapers/game_details_scrape.py +++ b/src/scrapers/game_details_scrape.py @@ -1,6 +1,7 @@ import re import requests from bs4 import BeautifulSoup +from urllib.parse import urljoin from src.utils.constants import * def clean_name(name): @@ -28,7 +29,8 @@ def fetch_page(url): def scrape_sidearm_story_recap(url): """ - Extract headline and published time from a Cornell Sidearm story/recap page + Extract headline, published time, and primary image from a Cornell Sidearm + story/recap page. """ if not url: return {} @@ -55,7 +57,15 @@ def scrape_sidearm_story_recap(url): pmeta = soup.find("meta", property="article:published_time") if pmeta and pmeta.get("content"): published_at = pmeta["content"].strip() - out = {} + image = soup.select_one(".sidearm-story-template-media img") + image_src = image.get("src") if image else None + out = { + "recap_article_image": ( + urljoin(f"{BASE_URL.rstrip('/')}/", image_src) + if image_src + else None + ) + } if title: out["recap_article_title"] = title if published_at: @@ -330,4 +340,4 @@ def scrape_game(url, sport): 'scoring_summary': scoring_summary or [{"message": "No scoring events in this game."}] } - return {"error": "Sport parser not found"} \ No newline at end of file + return {"error": "Sport parser not found"} diff --git a/src/scrapers/games_scraper.py b/src/scrapers/games_scraper.py index ddd393c..f0d4355 100644 --- a/src/scrapers/games_scraper.py +++ b/src/scrapers/games_scraper.py @@ -1,13 +1,12 @@ import requests from bs4 import BeautifulSoup -from src.services import GameService, TeamService from src.utils.convert_to_utc import convert_to_utc from src.utils.constants import * from src.scrapers.game_details_scrape import scrape_game, scrape_sidearm_story_recap from src.utils.helpers import get_dominant_color, normalize_game_data, is_tournament_placeholder_team, is_cornell_loss import base64 +import logging import re -from src.database import db import threading @@ -53,24 +52,26 @@ def to_absolute_url(link): def parse_game_links(game_item): """ Parse link info for a schedule item. - Returns (recap_link, ticket_link). + Keep Box Score and Recap links separate because they are different detail + sources. A schedule row can expose either link or both links. """ - recap_link = None box_score_tag = game_item.select_one(BOX_SCORE_TAG) - if box_score_tag: - box_score_link = box_score_tag.get("href") - recap_link = to_absolute_url(box_score_link) - - # Many sports expose recap links here when there is no separate box score link. - if not recap_link: - recap_tag = game_item.select_one(RECAP_TAG) - if recap_tag: - recap_link = to_absolute_url(recap_tag.get("href")) + recap_tag = game_item.select_one(RECAP_TAG) ticket_link_tag = game_item.select_one(GAME_TICKET_LINK) - ticket_link = to_absolute_url(ticket_link_tag["href"]) if ticket_link_tag else None - - return recap_link, ticket_link + return { + "box_score_link": ( + to_absolute_url(box_score_tag.get("href")) if box_score_tag else None + ), + "recap_link": ( + to_absolute_url(recap_tag.get("href")) if recap_tag else None + ), + "ticket_link": ( + to_absolute_url(ticket_link_tag.get("href")) + if ticket_link_tag + else None + ), + } def fetch_game_schedule(): """ @@ -79,14 +80,14 @@ def fetch_game_schedule(): """ threads = [] - for sport, data in SPORT_URLS.items(): - url = SCHEDULE_PREFIX + sport + SCHEDULE_POSTFIX + for sport_slug, data in SPORT_URLS.items(): + url = SCHEDULE_PREFIX + sport_slug + SCHEDULE_POSTFIX # create thread for each sport thread = threading.Thread( target=parse_schedule_page, args=(url, data["sport"], data["gender"]), - name=f"Scraper-{sport}" + name=f"Scraper-{sport_slug}" ) thread.daemon = True threads.append(thread) @@ -103,6 +104,10 @@ def parse_schedule_page(url, sport, gender): sport (str): The sport of the games. gender (str): The gender of the games. """ + if sport not in SPORT_DETAIL_MODES: + logging.warning("Skipping unsupported sport schedule: %s", sport) + return + response = requests.get(url, headers=HTTP_REQUEST_HEADERS, timeout=30) soup = BeautifulSoup(response.content, "html.parser") @@ -161,36 +166,43 @@ def parse_schedule_page(url, sport, gender): else: game_data["result"] = None - recap_link, ticket_link = parse_game_links(game_item) - - # These sports use news/recap pages instead of parsable box-score HTML. - if sport in SPORTS_WITH_SIDEARM_STORY_RECAP or not recap_link: - game_data["box_score"] = None - game_data["score_breakdown"] = None - else: - game_details = scrape_game(recap_link, sport.lower()) - if game_details.get("error") == "Sport parser not found": - game_data["box_score"] = None - game_data["score_breakdown"] = None - else: + links = parse_game_links(game_item) + box_score_link = links["box_score_link"] + recap_link = links["recap_link"] + detail_mode = SPORT_DETAIL_MODES.get(sport) + + if not detail_mode: + logging.warning("Skipping unsupported sport: %s", sport) + continue + + game_data.update({ + "box_score": None, + "score_breakdown": None, + "recap_link": recap_link if detail_mode == DETAIL_MODE_RECAP_LINK else None, + "recap_article_title": None, + "recap_published_at": None, + "recap_article_image": None, + }) + + if detail_mode == DETAIL_MODE_BOX_SCORE and box_score_link: + game_details = scrape_game(box_score_link, sport.lower()) + if not game_details.get("error"): game_data["box_score"] = game_details.get("scoring_summary") game_data["score_breakdown"] = game_details.get("scores") if sport in ["Baseball", "Football", "Lacrosse"]: - location_data = game_data["location"].split("\n") if game_data["location"] else [""] - geo_location = location_data[0] - is_home_game = "Ithaca" in geo_location - - if is_home_game and game_data["box_score"]: + geo_location = (game_data["location"] or "").split("\n")[0] + if "Ithaca" in geo_location and game_data["box_score"]: for event in game_data["box_score"]: if "cor_score" in event and "opp_score" in event: - event["cor_score"], event["opp_score"] = event["opp_score"], event["cor_score"] - - if sport in SPORTS_WITH_SIDEARM_STORY_RECAP and recap_link: + event["cor_score"], event["opp_score"] = ( + event["opp_score"], + event["cor_score"], + ) + elif detail_mode == DETAIL_MODE_RECAP_LINK and recap_link: game_data.update(scrape_sidearm_story_recap(recap_link)) - game_data["ticket_link"] = ticket_link - game_data["recap_link"] = recap_link + game_data["ticket_link"] = links["ticket_link"] process_game_data(game_data) @@ -202,6 +214,8 @@ def process_game_data(game_data): game_data (dict): A dictionary containing the game data. """ + from src.services import GameService, TeamService + game_data = normalize_game_data(game_data) location_raw = game_data.get("location") or "" location_data = location_raw.split("\n") if location_raw else [""] @@ -311,11 +325,10 @@ def process_game_data(game_data): "state": state, "ticket_link": game_data["ticket_link"], "recap_link": game_data.get("recap_link"), + "recap_article_title": game_data.get("recap_article_title"), + "recap_published_at": game_data.get("recap_published_at"), + "recap_article_image": game_data.get("recap_article_image"), } - if "recap_article_title" in game_data: - updates["recap_article_title"] = game_data["recap_article_title"] - if "recap_published_at" in game_data: - updates["recap_published_at"] = game_data["recap_published_at"] current_team = TeamService.get_team_by_id(curr_game.opponent_id) if current_team and is_tournament_placeholder_team(current_team.name): @@ -344,6 +357,7 @@ def process_game_data(game_data): "recap_link": game_data.get("recap_link"), "recap_article_title": game_data.get("recap_article_title"), "recap_published_at": game_data.get("recap_published_at"), + "recap_article_image": game_data.get("recap_article_image"), } - GameService.create_game(game_data) \ No newline at end of file + GameService.create_game(game_data) diff --git a/src/types.py b/src/types.py index 743a286..0583e8c 100644 --- a/src/types.py +++ b/src/types.py @@ -91,6 +91,7 @@ class GameType(ObjectType): - `recap_link`: The recap/details link of the game. (optional) - `recap_article_title`: Headline from the recap story page when scraped (optional) - `recap_published_at`: Published date/time from the recap story page (optional) + - `recap_article_image`: Primary image URL from the recap story page (optional) """ id = String(required=False) @@ -111,8 +112,27 @@ class GameType(ObjectType): recap_link = String(required=False) recap_article_title = String(required=False) recap_published_at = String(required=False) + recap_article_image = String(required=False) def __init__( - self, id, city, date, gender, location, opponent_id, result, sport, state, time, box_score=None, score_breakdown=None, utc_date=None, ticket_link=None, recap_link=None, recap_article_title=None, recap_published_at=None + self, + id, + city, + date, + gender, + location, + opponent_id, + result, + sport, + state, + time, + box_score=None, + score_breakdown=None, + utc_date=None, + ticket_link=None, + recap_link=None, + recap_article_title=None, + recap_published_at=None, + recap_article_image=None, ): self.id = id self.city = city @@ -131,6 +151,7 @@ def __init__( self.recap_link = recap_link self.recap_article_title = recap_article_title self.recap_published_at = recap_published_at + self.recap_article_image = recap_article_image @staticmethod def team_to_team_type(team_obj): @@ -208,4 +229,4 @@ class ArticleType(ObjectType): def __init__(self, **kwargs): for key, value in kwargs.items(): - setattr(self, key, value) \ No newline at end of file + setattr(self, key, value) diff --git a/src/utils/constants.py b/src/utils/constants.py index ab6ea6c..1997408 100644 --- a/src/utils/constants.py +++ b/src/utils/constants.py @@ -19,19 +19,9 @@ "Accept-Language": "en-US,en;q=0.9", } -# Schedule rows for these sports link to Sidearm story pages (no box score); scrape recap headline/date the same way. -SPORTS_WITH_SIDEARM_STORY_RECAP = frozenset( - { - "Swimming & Diving", - "Track & Field", - "Wrestling", - "Golf", - "Polo", - "Fencing", - "Equestrian", - "Gymnastics" - } -) +# Supported game-detail modes. +DETAIL_MODE_BOX_SCORE = "box_score" +DETAIL_MODE_RECAP_LINK = "recap_link" # The tag for each game GAME_TAG = ".sidearm-schedule-game" @@ -117,27 +107,31 @@ LABEL_SCORING_SUMMARY = 'Scoring Summary' LABEL_CU = 'CU' -# The dictionary mapping sports urls to gender +# The dictionary mapping sports urls to gender. +# +# Cross Country, Rowing, Sailing, Squash, Tennis, Volleyball, and Sprint +# Football are intentionally commented out until they have a supported game +# detail source. They should not be scraped as games in the meantime. SPORT_URLS = { "baseball": {"sport": "Baseball", "gender": "Mens"}, "mens-basketball": {"sport": "Basketball", "gender": "Mens"}, - "mens-cross-country": {"sport": "Cross Country", "gender": "Mens"}, + # "mens-cross-country": {"sport": "Cross Country", "gender": "Mens"}, "football": {"sport": "Football", "gender": "Mens"}, "mens-golf": {"sport": "Golf", "gender": "Mens"}, "mens-ice-hockey": {"sport": "Ice Hockey", "gender": "Mens"}, "mens-lacrosse": {"sport": "Lacrosse", "gender": "Mens"}, "mens-polo": {"sport": "Polo", "gender": "Mens"}, - "rowing": {"sport": "Rowing - Heavyweight", "gender": "Mens"}, - "mens-rowing": {"sport": "Rowing - Lightweight", "gender": "Mens"}, + # "rowing": {"sport": "Rowing - Heavyweight", "gender": "Mens"}, + # "mens-rowing": {"sport": "Rowing - Lightweight", "gender": "Mens"}, "mens-soccer": {"sport": "Soccer", "gender": "Mens"}, - "sprint-football": {"sport": "Sprint Football", "gender": "Mens"}, - "mens-squash": {"sport": "Squash", "gender": "Mens"}, + # "sprint-football": {"sport": "Sprint Football", "gender": "Mens"}, + # "mens-squash": {"sport": "Squash", "gender": "Mens"}, "mens-swimming-and-diving": {"sport": "Swimming & Diving", "gender": "Mens"}, - "mens-tennis": {"sport": "Tennis", "gender": "Mens"}, + # "mens-tennis": {"sport": "Tennis", "gender": "Mens"}, "mens-track-and-field": {"sport": "Track & Field", "gender": "Mens"}, "wrestling": {"sport": "Wrestling", "gender": "Mens"}, "womens-basketball": {"sport": "Basketball", "gender": "Womens"}, - "womens-cross-country": {"sport": "Cross Country", "gender": "Womens"}, + # "womens-cross-country": {"sport": "Cross Country", "gender": "Womens"}, "equestrian": {"sport": "Equestrian", "gender": "Womens"}, "fencing": {"sport": "Fencing", "gender": "Womens"}, "field-hockey": {"sport": "Field Hockey", "gender": "Womens"}, @@ -145,15 +139,35 @@ "womens-ice-hockey": {"sport": "Ice Hockey", "gender": "Womens"}, "womens-lacrosse": {"sport": "Lacrosse", "gender": "Womens"}, "womens-polo": {"sport": "Polo", "gender": "Womens"}, - "womens-rowing": {"sport": "Rowing ", "gender": "Womens"}, - "womens-sailing": {"sport": "Sailing", "gender": "Womens"}, + # "womens-rowing": {"sport": "Rowing", "gender": "Womens"}, + # "womens-sailing": {"sport": "Sailing", "gender": "Womens"}, "womens-soccer": {"sport": "Soccer", "gender": "Womens"}, "softball": {"sport": "Softball", "gender": "Womens"}, - "womens-squash": {"sport": "Squash", "gender": "Womens"}, + # "womens-squash": {"sport": "Squash", "gender": "Womens"}, "womens-swimming-and-diving": {"sport": "Swimming & Diving", "gender": "Womens"}, - "womens-tennis": {"sport": "Tennis", "gender": "Womens"}, + # "womens-tennis": {"sport": "Tennis", "gender": "Womens"}, "womens-track-and-field": {"sport": "Track & Field", "gender": "Womens"}, - "womens-volleyball": {"sport": "Volleyball", "gender": "Womens"}, + # "womens-volleyball": {"sport": "Volleyball", "gender": "Womens"}, +} + +# The only source of truth for how an active sport's game details are scraped. +SPORT_DETAIL_MODES = { + "Baseball": DETAIL_MODE_BOX_SCORE, + "Basketball": DETAIL_MODE_BOX_SCORE, + "Football": DETAIL_MODE_BOX_SCORE, + "Ice Hockey": DETAIL_MODE_BOX_SCORE, + "Field Hockey": DETAIL_MODE_BOX_SCORE, + "Lacrosse": DETAIL_MODE_BOX_SCORE, + "Soccer": DETAIL_MODE_BOX_SCORE, + "Softball": DETAIL_MODE_BOX_SCORE, + "Swimming & Diving": DETAIL_MODE_RECAP_LINK, + "Track & Field": DETAIL_MODE_RECAP_LINK, + "Wrestling": DETAIL_MODE_RECAP_LINK, + "Golf": DETAIL_MODE_RECAP_LINK, + "Polo": DETAIL_MODE_RECAP_LINK, + "Fencing": DETAIL_MODE_RECAP_LINK, + "Equestrian": DETAIL_MODE_RECAP_LINK, + "Gymnastics": DETAIL_MODE_RECAP_LINK, } IMAGE_BASE_URL = ( From 110e8ef4182641f777eddd8fd62959b153b8bfe8 Mon Sep 17 00:00:00 2001 From: claiireyu Date: Wed, 2 Sep 2026 23:06:11 -0400 Subject: [PATCH 3/4] fix(softball): apply scraper feedback Restore softball scoring parsing, preserve inning periods, normalize schedule results, and match updates by concrete game time. --- src/repositories/game_repository.py | 41 ++++++++++++++++++----------- src/scrapers/game_details_scrape.py | 29 ++++++++++++++++++++ src/scrapers/games_scraper.py | 9 ++++--- src/services/game_service.py | 10 +++---- 4 files changed, 65 insertions(+), 24 deletions(-) diff --git a/src/repositories/game_repository.py b/src/repositories/game_repository.py index 62a19d2..134aa15 100644 --- a/src/repositories/game_repository.py +++ b/src/repositories/game_repository.py @@ -11,6 +11,14 @@ logger = logging.getLogger(__name__) +def _time_for_lookup(time): + """Return whether a concrete time should be included in a game lookup.""" + if time is None: + return False + value = str(time).strip() + return bool(value) and value not in ("TBD", "TBA") + + class GameRepository: @staticmethod def find_all(limit=100, offset=0): @@ -103,24 +111,23 @@ def find_by_data(city, date, gender, location, opponent_id, sport, state, time): return Game.from_dict(game_data) if game_data else None @staticmethod - def find_by_key_fields(city, date, gender, location, opponent_id, sport, state): + def find_by_key_fields(city, date, gender, location, opponent_id, sport, state, time=None): """ - Find games without time for duplicate games + Find a game by its key fields, including a concrete time when available. """ game_collection = db["game"] - games = list( - game_collection.find( - { - "city": city, - "date": date, - "gender": gender, - "location": location, - "opponent_id": opponent_id, - "sport": sport, - "state": state, - } - ) - ) + base = { + "city": city, + "date": date, + "gender": gender, + "location": location, + "opponent_id": opponent_id, + "sport": sport, + "state": state, + } + if _time_for_lookup(time): + base["time"] = time + games = list(game_collection.find(base)) if not games: return None @@ -131,7 +138,7 @@ def find_by_key_fields(city, date, gender, location, opponent_id, sport, state): return [Game.from_dict(game) for game in games] @staticmethod - def find_by_tournament_key_fields(city, date, gender, location, sport, state): + def find_by_tournament_key_fields(city, date, gender, location, sport, state, time=None): """ Find tournament games by location and date (excluding opponent_id). This is used when we need to find a tournament game that might have a placeholder team. @@ -145,6 +152,8 @@ def find_by_tournament_key_fields(city, date, gender, location, sport, state): "gender": gender, "sport": sport, } + if _time_for_lookup(time): + query["time"] = time # For city, state, and location, use flexible matching # This allows finding games even when TBD/TBA values change to real values diff --git a/src/scrapers/game_details_scrape.py b/src/scrapers/game_details_scrape.py index efa477d..2f09f64 100644 --- a/src/scrapers/game_details_scrape.py +++ b/src/scrapers/game_details_scrape.py @@ -100,6 +100,34 @@ def extract_teams_and_scores(box_score_section, sport): return team_names, period_scores +def softball_summary(box_score_section): + summary = [] + scoring_section = box_score_section.find(TAG_SECTION, {ATTR_ARIA_LABEL: LABEL_SCORING_SUMMARY}) + if scoring_section: + scoring_rows = scoring_section.find(TAG_TBODY) + if scoring_rows: + for row in scoring_rows.find_all(TAG_TR): + team = row.find_all(TAG_TD)[0].find(TAG_IMG)[ATTR_ALT] + inning = row.find_all(TAG_TD)[3].text.strip() + desc_cell = row.find_all(TAG_TD)[4] + span = desc_cell.find(TAG_SPAN) + if span: + span.extract() + desc = desc_cell.get_text(strip=True) + cornell_score = int(row.find_all(TAG_TD)[5].get_text(strip=True) or 0) + opp_score = int(row.find_all(TAG_TD)[6].get_text(strip=True) or 0) + summary.append({ + 'team': team, + 'period': inning, + 'inning': inning, + 'description': desc, + 'cor_score': cornell_score, + 'opp_score': opp_score + }) + if not summary: + summary = [{"message": "No scoring events in this game."}] + return summary + def soccer_summary(box_score_section): summary = [] scoring_section = box_score_section.find(TAG_SECTION, {ATTR_ARIA_LABEL: LABEL_SCORING_SUMMARY}) @@ -320,6 +348,7 @@ def scrape_game(url, sport): 'field hockey': (lambda: extract_teams_and_scores(box_score_section, 'field hockey'), field_hockey_summary), 'lacrosse': (lambda: extract_teams_and_scores(box_score_section, 'lacrosse'), lacrosse_summary), 'baseball': (lambda: extract_teams_and_scores(box_score_section, 'baseball'), baseball_summary), + 'softball': (lambda: extract_teams_and_scores(box_score_section, 'softball'), softball_summary), 'basketball': (lambda: extract_teams_and_scores(box_score_section, 'basketball'), lambda _: []), } diff --git a/src/scrapers/games_scraper.py b/src/scrapers/games_scraper.py index f0d4355..ab6b51d 100644 --- a/src/scrapers/games_scraper.py +++ b/src/scrapers/games_scraper.py @@ -162,7 +162,8 @@ def parse_schedule_page(url, sport, gender): result_tag = game_item.select_one(RESULT_TAG) if result_tag: - game_data["result"] = result_tag.text.strip().replace("\n", "") + raw = result_tag.get_text(" ", strip=True) + game_data["result"] = re.sub(r"\s+", " ", raw).strip() else: game_data["result"] = None @@ -293,7 +294,8 @@ def process_game_data(game_data): game_data["gender"], location, game_data["sport"], - state + state, + game_time, ) # If no tournament game found, try the regular lookup with opponent_id @@ -305,7 +307,8 @@ def process_game_data(game_data): location, team.id, game_data["sport"], - state + state, + game_time, ) if isinstance(curr_game, list): diff --git a/src/services/game_service.py b/src/services/game_service.py index c7c4721..35edccc 100644 --- a/src/services/game_service.py +++ b/src/services/game_service.py @@ -74,22 +74,22 @@ def get_game_by_data(city, date, gender, location, opponent_id, sport, state, ti ) @staticmethod - def get_game_by_key_fields(city, date, gender, location, opponent_id, sport, state): + def get_game_by_key_fields(city, date, gender, location, opponent_id, sport, state, time=None): """ - Retrieve a game by its essential fields, ignoring time + Retrieve a game by its key fields, including time when available. """ return GameRepository.find_by_key_fields( - city, date, gender, location, opponent_id, sport, state + city, date, gender, location, opponent_id, sport, state, time ) @staticmethod - def get_game_by_tournament_key_fields(city, date, gender, location, sport, state): + def get_game_by_tournament_key_fields(city, date, gender, location, sport, state, time=None): """ Retrieve a tournament game by location and date (excluding opponent_id). This is used when we need to find a tournament game that might have a placeholder team. """ return GameRepository.find_by_tournament_key_fields( - city, date, gender, location, sport, state + city, date, gender, location, sport, state, time ) @staticmethod From fb4dfef0b89b8aa1d7e3ac1b34f829605d669682 Mon Sep 17 00:00:00 2001 From: claiireyu Date: Wed, 2 Sep 2026 23:17:34 -0400 Subject: [PATCH 4/4] fix(scores): keep inning events under period --- src/scrapers/game_details_scrape.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/scrapers/game_details_scrape.py b/src/scrapers/game_details_scrape.py index 2f09f64..6ae5798 100644 --- a/src/scrapers/game_details_scrape.py +++ b/src/scrapers/game_details_scrape.py @@ -119,7 +119,6 @@ def softball_summary(box_score_section): summary.append({ 'team': team, 'period': inning, - 'inning': inning, 'description': desc, 'cor_score': cornell_score, 'opp_score': opp_score @@ -358,10 +357,6 @@ def scrape_game(url, sport): if extract_teams_func and summary_func: team_names, scores = extract_teams_func() scoring_summary = summary_func(box_score_section) - - for event in scoring_summary: - if not event.get("time") and event.get("period"): - event["time"] = event["period"] return { 'teams': team_names,