streaming zip files
ben/archives/pipeline/head This commit looks good Details

main
parent 18957b9869
commit f521df810a

3
.gitignore vendored

@ -154,4 +154,5 @@ cython_debug/
prod-archives.toml
*.json
temp.toml
temp.toml
dev.toml

@ -1,10 +1,14 @@
import configparser
import os
import sys
from loguru import logger
def read_config(path):
if not os.path.isfile(path):
# or: FileNotFoundError
raise SystemExit(f"config file not found: {path}")
conf = configparser.ConfigParser()
conf.read(path)
return conf

@ -1,11 +1,13 @@
import os
from datetime import datetime
from pathlib import Path
from urllib.parse import quote, urljoin
import aiofiles
import humanize
from aiohttp.web import HTTPBadRequest, HTTPNotFound
from loguru import logger
from zipstream import AioZipStream
from archives.ducache import DuCache
@ -17,23 +19,23 @@ class FileBrowser:
self.du_file = self.config["du_cache_file"]
self.du_cache = DuCache(self.du_file, self.path)
def zip_name(self, rel_path):
p = Path(rel_path)
return f"{p.stem}.zip"
def full_path(self, rel_path):
return os.path.join(self.path, rel_path)
def list_dir(self, rel_path, sort_by="name", filter_list=None):
files = list(self.browse_dir(rel_path))
if filter_list is not None:
files = [a for a in files if a["name"] in filter_list]
by_key = sorted(
list(files), key=lambda a: a[sort_by], reverse=sort_by != "name"
)
return sorted(by_key, key=lambda a: a["is_dir"], reverse=True)
def is_relative(self, rel_path):
local_path = self.full_path(rel_path)
return Path(local_path).is_relative_to(self.path)
def isdir(self, rel_path):
return os.path.isdir(self.full_path(rel_path))
def isfile(self, rel_path):
return os.path.isfile(self.full_path(rel_path))
def exists(self, rel_path):
local_path = self.full_path(rel_path)
e = os.path.exists(local_path)
@ -48,6 +50,7 @@ class FileBrowser:
return humanize.naturalsize(size, binary=binary)
async def file_sender(self, rel_path):
# has been checked if its from the request object
if rel_path.startswith("/"):
raise HTTPBadRequest
@ -56,9 +59,41 @@ class FileBrowser:
while chunk := await f.read(2**16):
yield chunk
def files_to_zip(self, local_path):
for f in Path(local_path).glob("**/*"):
if self.isfile(f):
item = {
'file': str(f),
'name': str(f.relative_to(local_path)),
'compression': 'deflate',
'is_dir': self.isdir(str(f))
}
yield item
async def zip_sender(self, rel_path):
local_path = self.full_path(rel_path)
files = self.files_to_zip(local_path)
aiozip = AioZipStream(files, chunksize=2**16)
async for chunk in aiozip.stream():
yield chunk
def list_dir(self, rel_path, sort_by="name", filter_list=None):
files = list(self.browse_dir(rel_path))
if filter_list is not None:
files = [a for a in files if a["name"] in filter_list]
by_key = sorted(
list(files), key=lambda a: a[sort_by], reverse=sort_by != "name"
)
return sorted(by_key, key=lambda a: a["is_dir"], reverse=True)
def browse_dir(self, rel_path, items=None):
if rel_path.startswith("/"):
raise HTTPBadRequest
if not self.isdir(rel_path):
raise HTTPBadRequest
p = self.full_path(rel_path)
if not os.path.isdir(p):

@ -68,10 +68,12 @@ class AsyncServer:
_path = request.match_info.get("path", "")
logger.debug(f"path: '{_path}'")
if _path.startswith("/"):
if _path.startswith("/") or not self.browser.is_relative(_path):
logger.error(f"weird path: '{_path}'")
raise HTTPBadRequest
# archives_path has been checked for leading / if used
# from the request object
request["archives_path"] = _path
response = await handler(request)
@ -113,23 +115,35 @@ class AsyncServer:
file_path = request["archives_path"]
file_name = os.path.basename(file_path)
sender = self.browser.file_sender(file_path)
return Response(
body=self.browser.file_sender(file_path),
body=sender,
headers={
"Content-disposition": f"attachment; filename={file_name}"
},
)
})
async def download_zip(self, request):
dir_path = request['archives_path']
zip_name = self.browser.zip_name(dir_path)
return Response(
body=self.browser.zip_sender(dir_path),
headers={
"Content-disposition": f"attachment; filename={zip_name}"
})
@template("files.html.j2")
async def archives(self, request):
# make a chooser
if not self.browser.exists(request["archives_path"]):
raise HTTPNotFound
if not self.browser.isdir(request["archives_path"]):
if self.browser.isfile(request["archives_path"]):
return await self.download(request)
if 'zip' in request.query:
return await self.download_zip(request)
try:
sort_by = request.query.get("sort_by", "name")
items = self.browser.list_dir(

@ -1,6 +1,12 @@
{% extends "base.html.j2" %}
{% block content %}
{#
# icons: https://store.kde.org/p/1228310/
# https://github.com/MarianArlt/pixel-perfect-folder-icons
# https://github.com/alvatip/Nordzy-icon
#}
<div class="terminal">
<div class="shellcommand">
<span class="shellps1">{{ username }}<b>@</b>{{ domain }}:{{ ps1_path }}<b>$</b></span>
@ -16,13 +22,21 @@
<th>
</th>
<th class="filename">
name
<a href="?sort_by=name">
name
</a>
</th>
<th>
</th>
<th>
date
<a href="?sort_by=date">
date
</a>
</th>
<th>
size
<a href="?sort_by=size">
size
</a>
</th>
</tr>
</thead>
@ -39,6 +53,8 @@
</td>
<td>
</td>
<td>
</td>
</tr>
{% endif %}
@ -47,11 +63,18 @@
<td>
{% if item.is_dir -%}
<img src="{{ base_url }}img/22px-folder.svg" class="listicon">
{% else %}
<img src="{{ base_url }}img/text-x-generic.svg" class="listicon">
{% endif %}
</td>
<td>
<a href="{{ item.href }}">{{ item.name }}</a>
</td>
<td>
{% if item.is_dir %}
[<a href="{{ item.href }}?zip">zip</a>]
{% endif %}
</td>
<td>
{{ " " }} {{ item.date }} {{ " " }}
</td>

14
poetry.lock generated

@ -49,6 +49,14 @@ python-versions = ">=3.6"
[package.dependencies]
frozenlist = ">=1.1.0"
[[package]]
name = "aiozipstream"
version = "0.4"
description = "Creating zip files on the fly"
category = "main"
optional = false
python-versions = "*"
[[package]]
name = "async-timeout"
version = "4.0.2"
@ -423,7 +431,7 @@ multidict = ">=4.0"
[metadata]
lock-version = "1.1"
python-versions = "^3.9"
content-hash = "6076b2eeffe041fdcd450f911f7e7d7400ab37182950b9593ef7246efc4d86c1"
content-hash = "9b5341a92391d20801dd24d11d8c07b28b70e2470aa1931c88870a7f27ce31de"
[metadata.files]
aiofiles = [
@ -512,6 +520,10 @@ aiosignal = [
{file = "aiosignal-1.2.0-py3-none-any.whl", hash = "sha256:26e62109036cd181df6e6ad646f91f0dcfd05fe16d0cb924138ff2ab75d64e3a"},
{file = "aiosignal-1.2.0.tar.gz", hash = "sha256:78ed67db6c7b7ced4f98e495e572106d5c432a93e1ddd1bf475e1dc05f5b7df2"},
]
aiozipstream = [
{file = "aiozipstream-0.4-py2.py3-none-any.whl", hash = "sha256:a58bad8c75aba319c07bd3d817da7caec7417c1eb4f4c692e00b173fb9ded9c6"},
{file = "aiozipstream-0.4.tar.gz", hash = "sha256:ccc5cec35c2580b8a13185c916b1581bfcb4278ddf6ea3f7f834b6c9c47d6c61"},
]
async-timeout = [
{file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"},
{file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"},

@ -13,6 +13,7 @@ aiohttp = "^3.8.1"
aiohttp-jinja2 = "^1.5"
humanize = "^3.14.0"
aiofiles = "^0.8.0"
aiozipstream = "^0.4"
[tool.poetry.dev-dependencies]
pytest = "^5.2"

Loading…
Cancel
Save