50 lines
1.4 KiB
Python
50 lines
1.4 KiB
Python
import requests
|
|
import base64
|
|
import io
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
def url_to_filename(url):
|
|
return base64.b64encode(url.encode('utf8')).decode('utf8')
|
|
|
|
def filename_to_url(filename):
|
|
return base64.b64decode(filename.encode('utf8')).decode('utf8')
|
|
|
|
class RequestCache:
|
|
def __init__(self, cache_dir):
|
|
self.cache_dir = cache_dir
|
|
self.cache = {}
|
|
folder = Path(self.cache_dir)
|
|
if not folder.is_dir():
|
|
folder.mkdir(parents=True)
|
|
files_list = [p for p in folder.iterdir() if p.is_file()]
|
|
for file in files_list:
|
|
fname = file.name
|
|
unbase = filename_to_url(fname)
|
|
self.cache[fname] = unbase
|
|
|
|
def get_cache(self, url):
|
|
filename = url_to_filename(url)
|
|
print("looking for filename:", filename)
|
|
if filename in self.cache:
|
|
print("fetching local copy")
|
|
|
|
return open(Path(self.cache_dir) / filename,'r').read()
|
|
else:
|
|
try:
|
|
print("have to go to internet to fetch")
|
|
response = requests.get(url)
|
|
data = response.text
|
|
file_path = Path(self.cache_dir) / filename
|
|
with open(file_path, 'w') as f:
|
|
f.write(data)
|
|
print("updating cache dict")
|
|
self.cache[filename] = url
|
|
return data
|
|
except Exception as e:
|
|
print(f"Error {str(e)}")
|
|
|
|
|
|
|
|
|