ORR rail statistics data portal

curl 'https://dataportal.orr.gov.uk/statistics/usage/freight-rail-usage-and-performance/table-1314-freight-moved-by-commodity-periodic-1/'

"""ORR periodic freight-by-commodity table -> pandas -> Parquet. No key."""
from html.parser import HTMLParser
import requests
import pandas as pd

URL = "https://dataportal.orr.gov.uk/statistics/usage/freight-rail-usage-and-performance/table-1314-freight-moved-by-commodity-periodic-1/"

class TableParser(HTMLParser):
    def __init__(self):
        super().__init__()
        self.in_table = False
        self.in_cell = False
        self.row = []
        self.rows = []
        self.text = []
    def handle_starttag(self, tag, attrs):
        attrs = dict(attrs)
        if tag == "table" and attrs.get("id") == "reportTable":
            self.in_table = True
        elif self.in_table and tag == "tr":
            self.row = []
        elif self.in_table and tag in {"th", "td"}:
            self.in_cell = True
            self.text = []
    def handle_data(self, data):
        if self.in_cell:
            self.text.append(data)
    def handle_endtag(self, tag):
        if self.in_table and tag in {"th", "td"}:
            self.row.append(" ".join("".join(self.text).split()))
            self.in_cell = False
        elif self.in_table and tag == "tr" and self.row:
            self.rows.append(self.row)
        elif self.in_table and tag == "table":
            self.in_table = False

r = requests.get(URL, timeout=60)
r.raise_for_status()
parser = TableParser()
parser.feed(r.text)
df = pd.DataFrame(parser.rows[1:], columns=parser.rows[0])
df.to_parquet("orr_freight_moved_by_commodity.parquet", index=False)
print(len(df), "rows", df["Time period"].iloc[0], "to", df["Time period"].iloc[-1],
      len(df.columns) - 1, "commodity columns")