From 60499557151b7075fc866d11152def7c090196ff Mon Sep 17 00:00:00 2001 From: admin Date: Thu, 26 Feb 2026 13:55:50 -0800 Subject: [PATCH] requestcache, getting a url will return the data if it has it, fetch and store and return it if not --- requestcache.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/requestcache.py b/requestcache.py index e69de29..5ff3d94 100644 --- a/requestcache.py +++ b/requestcache.py @@ -0,0 +1,36 @@ +import requests +import base64 +import io +import sys +from pathlib import Path + +class RequestCache: + def __init__(self, cache_dir): + self.cache_dir = cache_dir + self.cache = {} + folder = Path(self.cache_dir) + if not folder.is_dir(): + folder.mkdir(parents=True) + files_list = [p for p in folder.iterdir() if p.is_file()] + for file in files_list: + fname = file.name + unbase = base64.standard_b64decode(fname) + self.cache[unbase] = fname + + def get_cache(self, url): + b64 = base64.urlsafe_b64encode(url) + if b64 in self.cache: + return open(self.cache[b64],'r').read() + else: + try: + response = requests.get(url) + data = response.text + file_path = self.cache_dir / b64 + with open(file_path, 'w') as f: + f.write(data) + except Exception as e: + print(f"Error {str(e)}") + + + +