| 123456789101112131415161718192021222324252627282930313233 |
- import lzma
- import pathlib
- import time
- import typing
- import cbor2
- import httpx
- import urllib.parse
- if typing.TYPE_CHECKING:
- import httpx._types
- CACHE_DIR = pathlib.Path(__file__).parent / 'cache'
- client = httpx.Client(transport=httpx.HTTPTransport(http2=True, retries=2))
- def get(url: str, *, headers: httpx._types.HeaderTypes|None=None) -> typing.Any:
- parsed = urllib.parse.urlparse(url)
- assert parsed.hostname is not None
- cache_path = CACHE_DIR / parsed.hostname / (urllib.parse.quote(parsed.path.removeprefix('/'), safe='') + '.cbor.xz')
- try:
- if cache_path.stat().st_mtime > time.time() - 600: # less than 10 minutes old
- with lzma.open(cache_path, 'rb') as f:
- return cbor2.load(f)
- except FileNotFoundError:
- pass # fall through
- r = client.get(url, headers=headers).raise_for_status()
- cache_path.parent.mkdir(parents=True, exist_ok=True)
- with lzma.open(cache_path, 'wb') as f:
- data = r.json()
- cbor2.dump(data, f)
- return data
|