now it works
This commit is contained in:
parent
6049955715
commit
c4e1d0cbe9
@ -4,6 +4,12 @@ import io
|
|||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
def url_to_filename(url):
|
||||||
|
return base64.b64encode(url.encode('utf8')).decode('utf8')
|
||||||
|
|
||||||
|
def filename_to_url(filename):
|
||||||
|
return base64.b64decode(filename.encode('utf8')).decode('utf8')
|
||||||
|
|
||||||
class RequestCache:
|
class RequestCache:
|
||||||
def __init__(self, cache_dir):
|
def __init__(self, cache_dir):
|
||||||
self.cache_dir = cache_dir
|
self.cache_dir = cache_dir
|
||||||
@ -14,20 +20,27 @@ class RequestCache:
|
|||||||
files_list = [p for p in folder.iterdir() if p.is_file()]
|
files_list = [p for p in folder.iterdir() if p.is_file()]
|
||||||
for file in files_list:
|
for file in files_list:
|
||||||
fname = file.name
|
fname = file.name
|
||||||
unbase = base64.standard_b64decode(fname)
|
unbase = filename_to_url(fname)
|
||||||
self.cache[unbase] = fname
|
self.cache[fname] = unbase
|
||||||
|
|
||||||
def get_cache(self, url):
|
def get_cache(self, url):
|
||||||
b64 = base64.urlsafe_b64encode(url)
|
filename = url_to_filename(url)
|
||||||
if b64 in self.cache:
|
print("looking for filename:", filename)
|
||||||
return open(self.cache[b64],'r').read()
|
if filename in self.cache:
|
||||||
|
print("fetching local copy")
|
||||||
|
|
||||||
|
return open(Path(self.cache_dir) / filename,'r').read()
|
||||||
else:
|
else:
|
||||||
try:
|
try:
|
||||||
|
print("have to go to internet to fetch")
|
||||||
response = requests.get(url)
|
response = requests.get(url)
|
||||||
data = response.text
|
data = response.text
|
||||||
file_path = self.cache_dir / b64
|
file_path = Path(self.cache_dir) / filename
|
||||||
with open(file_path, 'w') as f:
|
with open(file_path, 'w') as f:
|
||||||
f.write(data)
|
f.write(data)
|
||||||
|
print("updating cache dict")
|
||||||
|
self.cache[filename] = url
|
||||||
|
return data
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"Error {str(e)}")
|
print(f"Error {str(e)}")
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user