import requests import base64 import io import sys from pathlib import Path def url_to_filename(url): return base64.b64encode(url.encode('utf8')).decode('utf8') def filename_to_url(filename): return base64.b64decode(filename.encode('utf8')).decode('utf8') class RequestCache: def __init__(self, cache_dir): self.cache_dir = cache_dir self.cache = {} folder = Path(self.cache_dir) if not folder.is_dir(): folder.mkdir(parents=True) files_list = [p for p in folder.iterdir() if p.is_file()] for file in files_list: fname = file.name unbase = filename_to_url(fname) self.cache[fname] = unbase def get_cache(self, url): filename = url_to_filename(url) print("looking for filename:", filename) if filename in self.cache: print("fetching local copy") return open(Path(self.cache_dir) / filename,'r').read() else: try: print("have to go to internet to fetch") response = requests.get(url) data = response.text file_path = Path(self.cache_dir) / filename with open(file_path, 'w') as f: f.write(data) print("updating cache dict") self.cache[filename] = url return data except Exception as e: print(f"Error {str(e)}")