solderpunk
/
AV-98
9
21
Fork 15
Code Issues 4 Pull Requests 4 Releases Wiki Activity

Streaming servers without content-length are closed after 5Mo of download

This commit is contained in:
Lionel Dricot 2022-03-24 22:00:04 +01:00
parent c2879828d6
commit 05da20801c
2 changed files with 22 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG
View File

@ -1,6 +1,7 @@
# Offpunk History # Offpunk History
## 1.3 - Unreleased ## 1.3 - Unreleased
- Streaming URL without valid content-length are now closed after 5Mo of download (thanks to Eoin Carney for reporting the issue)
- Fixed a crash when the cache is already a dir inside a dir. - Fixed a crash when the cache is already a dir inside a dir.
## 1.2 - March 24th 2022 ## 1.2 - March 24th 2022

28
offpunk.py
View File

@ -1808,7 +1808,7 @@ class GeminiClient(cmd.Cmd):
"tls_mode" : "tofu", "tls_mode" : "tofu",
"archives_size" : 200, "archives_size" : 200,
"history_size" : 200, "history_size" : 200,
"max_size_download" : 20, "max_size_download" : 10,
} }
global TERM_WIDTH global TERM_WIDTH
TERM_WIDTH = self.options["width"] TERM_WIDTH = self.options["width"]
@ -1969,9 +1969,17 @@ class GeminiClient(cmd.Cmd):
print("You can use the ! command to specify another handler program or pipeline.") print("You can use the ! command to specify another handler program or pipeline.")
def _fetch_http(self,gi,max_length=None): def _fetch_http(self,gi,max_length=None):
def set_error(item,length,max_length):
err = "Size of %s is %s Mo\n"%(item.url,length)
err += "Offpunk only download automatically content under %s Mo\n" %(max_length/1000000)
err += "To retrieve this content anyway, type 'reload'."
item.set_error(err)
return item
header = {} header = {}
header["User-Agent"] = "Offpunk browser v%s"%_VERSION header["User-Agent"] = "Offpunk browser v%s"%_VERSION
with requests.get(gi.url,headers=header, stream=True) as response: with requests.get(gi.url,headers=header, stream=True,timeout=5) as response:
#print("This is header for %s"%gi.url)
#print(response.headers)
if "content-type" in response.headers: if "content-type" in response.headers:
mime = response.headers['content-type'] mime = response.headers['content-type']
else: else:
@ -1982,11 +1990,17 @@ class GeminiClient(cmd.Cmd):
length = 0 length = 0
if max_length and length > max_length: if max_length and length > max_length:
response.close() response.close()
err = "Size of %s is %s Mo\n"%(gi.url,length/1000000) return set_error(gi,str(length/1000000),max_length)
err += "Offpunk only download automatically content under %s Mo\n" %(max_length/1000000) elif max_length and length == 0:
err += "To retrieve this content anyway, type 'reload'." body = b''
gi.set_error(err) for r in response.iter_content():
return gi body += r
#We divide max_size for streamed content
#in order to catch them faster
if sys.getsizeof(body) > max_length/2:
response.close()
return set_error(gi,"streaming",max_length)
response.close()
else: else:
body = response.content body = response.content
response.close() response.close()