import lzma import pathlib import time import typing import cbor2 import httpx import urllib.parse if typing.TYPE_CHECKING: import httpx._types CACHE_DIR = pathlib.Path(__file__).parent / 'cache' client = httpx.Client(transport=httpx.HTTPTransport(http2=True, retries=2)) def get(url: str, *, headers: httpx._types.HeaderTypes|None=None) -> typing.Any: parsed = urllib.parse.urlparse(url) assert parsed.hostname is not None cache_path = CACHE_DIR / parsed.hostname / (urllib.parse.quote(parsed.path.removeprefix('/'), safe='') + '.cbor.xz') try: if cache_path.stat().st_mtime > time.time() - 600: # less than 10 minutes old with lzma.open(cache_path, 'rb') as f: return cbor2.load(f) except FileNotFoundError: pass # fall through r = client.get(url, headers=headers).raise_for_status() cache_path.parent.mkdir(parents=True, exist_ok=True) with lzma.open(cache_path, 'wb') as f: data = r.json() cbor2.dump(data, f) return data