Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use the block size determined by asyncssh if nothing is specified #52

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 28 additions & 16 deletions sshfs/file.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,32 +26,37 @@ def __init__(
self.mode = mode
self.max_requests = max_requests or _MAX_SFTP_REQUESTS

if block_size is None:
# "The OpenSSH SFTP server will close the connection
# if it receives a message larger than 256 KB, and
# limits read requests to returning no more than
# 64 KB."
#
# We are going to use the maximum block_size possible
# with a 16KB margin (so instead of sending 256 KB data,
# we'll send 240 KB + headers for write requests)

if self.readable():
block_size = READ_BLOCK_SIZE
else:
block_size = WRITE_BLOCK_SIZE

# The blocksize is often used with constructs like
# shutil.copyfileobj(src, dst, length=file.blocksize) and since we are
# using pipelining, we are going to reflect the total size rather than
# a size of chunk to our limits.
self.blocksize = block_size * self.max_requests
self.blocksize = (
None if block_size is None else block_size * self.max_requests
)

self.kwargs = kwargs

self._file = sync(self.loop, self._open_file)
self._closed = False

def _determine_block_size(self, channel):
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should it be cached? should it be async?

# Use the asyncssh block sizes to ensure the best performance.
limits = getattr(channel, "limits", None)
if limits:
if self.readable():
return limits.max_read_len
return limits.max_write_len

# "The OpenSSH SFTP server will close the connection
# if it receives a message larger than 256 KB, and
# limits read requests to returning no more than
# 64 KB."
#
# We are going to use the maximum block_size possible
# with a 16KB margin (so instead of sending 256 KB data,
# we'll send 240 KB + headers for write requests)
return READ_BLOCK_SIZE if self.readable() else WRITE_BLOCK_SIZE

@wrap_exceptions
async def _open_file(self):
# TODO: this needs to keep a reference to the
Expand All @@ -61,6 +66,10 @@ async def _open_file(self):
# it's operations but the pool it thinking this
# channel is freed.
async with self.fs._pool.get() as channel:
if self.blocksize is None:
self.blocksize = (
self._determine_block_size(channel) * self.max_requests
)
return await channel.open(
self.path,
self.mode,
Expand All @@ -81,6 +90,9 @@ async def _open_file(self):
def readable(self):
return "r" in self.mode

def seekable(self):
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

needs rebase?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes

return "r" in self.mode or "w" in self.mode

def writable(self):
return not self.readable()

Expand Down
Loading