2020 Week 3 General Thread

#!/usr/bin/env perl
use Getopt::Long;
use Mojo::UserAgent;

use strict;
use warnings;

use constant SCRAPE_URL =>
  'https://www.sportingnews.com/us/nfl/news/nfl-schedule-2020-dates-times-channels-week-by-week/112tca2d0jcy91t47yhzemrrhp';

my %TEAM_MAP = (
    'Jacksonville Jaguars' => 'Jacksonville',
    'Miami Dolphins'       => 'Miami',
    'Atlanta Falcons'      => 'Atlanta',
    'Chicago Bears'        => 'Chicago',
    'Buffalo Bills'        => 'Buffalo',
    'Los Angeles Rams'     => 'Los Angeles Rams',
    'Cleveland Browns'     => 'Cleveland',
    'Washington Redskins'  => 'Washington',
    'Minnesota Vikings'    => 'Minnesota',
    'Tennessee Titans'     => 'Tennessee',
    'New England Patriots' => 'New England',
    'Las Vegas Raiders'    => 'Las Vegas',
    'New York Giants'      => 'New York Giants',
    'San Francisco 49ers'  => 'San Francisco',
    'Philadelphia Eagles'  => 'Philadelphia',
    'Cincinnati Bengals'   => 'Cincinnati',
    'Pittsburgh Steelers'  => 'Pittsburgh',
    'Houston Texans'       => 'Houston',
    'Indianapolis Colts'   => 'Indianapolis',
    'New York Jets'        => 'New York Jets',
    'Los Angeles Chargers' => 'Los Angeles Chargers',
    'Carolina Panthers'    => 'Carolina',
    'Arizona Cardinals'    => 'Arizona',
    'Detroit Lions'        => 'Detroit',
    'Denver Broncos'       => 'Denver',
    'Tampa Bay Buccaneers' => 'Tampa Bay',
    'Seattle Seahawks'     => 'Seattle',
    'Dallas Cowboys'       => 'Dallas',
    'New Orleans Saints'   => 'New Orleans',
    'Green Bay Packers'    => 'Green Bay',
    'Baltimore Ravens'     => 'Baltimore',
    'Kansas City Chiefs'   => 'Kansas City',
);

my %TV_MAP = ('NFLN' => 'NFL Network',);

sub handle_week($$) {
    my ($sched_hashref, $e) = @_;
    die "Invalid week header" unless $e->all_text =~ /^Week (\d+)$/;
    $sched_hashref->{$1} = [];
    return $1;
}

sub handle_gameday($$$) {
    my ($sched_hashref, $current_week, $e) = @_;
    return $e->all_text();
}

sub handle_gametable($$$$) {
    my ($sched_hashref, $current_week, $current_gameday, $e) = @_;
    foreach my $kid ($e->children('table > tbody')->each) {
        foreach my $babby ($kid->children('tr')->each) {
            my ($teams, $time, $network) =
              map {$_->all_text} ($babby->children('td')->each);
            next if $teams =~ /^Game$/;
            my ($home, $away) = split(/\sat\s/, $teams);
            push @{ $sched_hashref->{$current_week} },
              {
                'gameday' => $current_gameday,
                'home'    => $home,
                'away'    => $away,
                'time'    => $time,
                'network' => $network
              };
        }
    }
}

sub scrape_page($) {
    my ($tgt_url) = @_;

    my $ua       = Mojo::UserAgent->new;
    my $res      = $ua->get(SCRAPE_URL)->result;
    my $gamedata = $res->dom->find(
        'body > main.main-content > section.grid > article.article > div > section.article-body ~ h2,h3.block-element__h3,ul.block-element__ul,table'
    );

    my $sched_hashref   = {};
    my $current_week    = -1;
    my $current_gameday = undef;

    foreach my $e ($gamedata->each) {
        if ($e->matches('h3')) {
            $current_week = handle_week($sched_hashref, $e);
        }
        elsif ($e->matches('ul')) {
            $current_gameday =
              handle_gameday($sched_hashref, $current_week, $e);
        }
        elsif ($e->matches('table')) {
            handle_gametable($sched_hashref, $current_week,
                $current_gameday, $e);
        }
        else {
            die "SHIT";
        }
    }

    return $sched_hashref;
}

sub print_schedule($$) {
    my ($sched_hashref, $week) = @_;
    die unless $sched_hashref->{$week};
    print "### Week $week\n";
    print join('|',
        '',            'Away',   'Home', 'Day/Time',
        'Broadcaster', 'Result', "\n");
    print join('|',
        '',            '----',   '----', '--------',
        '-----------', '------', "\n");
    foreach my $game (@{ $sched_hashref->{$week} }) {
        $game->{'time'} =~ s/p\.m\./PM/;
        print join('|',
            '',
            $TEAM_MAP{ $game->{'away'} } // '',
            $TEAM_MAP{ $game->{'home'} } // '',
            join(' @ ', $game->{'gameday'}, $game->{'time'}),
            $TEAM_MAP{ $game->{'network'} } // $game->{'network'},
            '',
            "\n");
    }
}

sub main() {
    my $week = 3;
    GetOptions("week=i", \$week)
      or die "Error in command line arguments";

    my $sched_hashref = scrape_page(SCRAPE_URL);
    print_schedule($sched_hashref, $week);
}

main();

I can probably consolidate some of those CSS selectors, but that requires thinking and it’s the weekend…

import argparse
import bs4
import re
import requests

SCRAPE_URL = 'https://www.sportingnews.com/us/nfl/news/nfl-schedule-2020-dates-times-channels-week-by-week/112tca2d0jcy91t47yhzemrrhp'
TEAM_REWRITE = {'Los Angeles Rams', 'New York Giants', 'New York Jets',
                'Los Angeles Chargers'}
NETWORK_REWRITE = {'NFLN': 'NFL Network'}


def handle_week(element, week_dict):
    m = re.match(r'^Week (\d+)$', element.string)
    if not m:
        print("FUCK")
        exit(1)

    week_dict[m.group(1)] = []
    return m.group(1)


def trim_team_name(team):
    if team not in TEAM_REWRITE:
        parts = team.split(" ")
        team = " ".join(parts[0:-1])

    return team


def handle_gametable(element, cur_week, cur_gameday, week_dict):
    cells = element.select('table > tbody tr > td')

    # Make sure we have a multiple of three cells
    if (len(cells) % 3 != 0):
        print("FUCK")
        exit(1)

    while(cells):
        teams, time, network = cells[0:3]
        if not re.match(r"^Game$", teams.string):
            home_team, away_team = map(trim_team_name,
                                       teams.string.split(' at '))

            network = network.string
            if network in NETWORK_REWRITE:
                network = NETWORK_REWRITE[network]

            p = re.compile(r'p\.m\.')
            time = ' @ '.join([cur_gameday, p.sub('PM', time.string)])

            gamedata = {'home': home_team,
                        'away': away_team,
                        'time': time,
                        'network': network}

            week_dict[cur_week].append(gamedata)
        cells = cells[3:]


def scrape_page(url):
    week_dict = {}

    r = requests.get(url)

    if not r.ok:
        print("FUCK")
        exit(1)

    soup = bs4.BeautifulSoup(r.text, 'html.parser')
    elements = soup.select('body > main.main-content > section.grid > article.article > div > section.article-body ~ h2,h3.block-element__h3,ul.block-element__ul,table')

    cur_week, cur_gameday = -1, None

    for element in elements:
        if (element.name == "h3"):
            cur_week = handle_week(element, week_dict)

        elif (element.name == "ul"):
            cur_gameday = element.text

        elif (element.name == "table"):
            handle_gametable(element, cur_week, cur_gameday, week_dict)

        else:
            print("FUCK")
            exit(1)

    return week_dict


def print_tz_markdown(week, gamedata):
    week = str(week)
    if week not in gamedata:
        print("FUCK")
        exit(1)

    print(f'### Week {week}')
    print('|'.join(['', 'Away', 'Home', 'Day/Time', 'Broadcaster', 'Result',
                    '']))
    print('|'.join(['', '----', '----', '--------', '-----------', '------',
                    '']))

    games = gamedata[week]
    for game in games:
        print('|'.join(['', game['away'], game['home'], game['time'],
                        game['network'], '', '']))


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Dump games for NFL threads')
    parser.add_argument('--week', type=int, default=3,
                        help="Week to scrape data for")
    args = parser.parse_args()

    gamedata = scrape_page(SCRAPE_URL)
    print_tz_markdown(args.week, gamedata)