Skip to content

Commit

Permalink
Merge pull request #157 from FrendsPlatform/issue-155
Browse files Browse the repository at this point in the history 
Bug fix for SFTP.UploadFiles: Added operations timeout handling for issue
  • Loading branch information
@ttossavainen
ttossavainen authored Jul 25, 2023
2 parents 8ad9e2d + ce4fc91 commit d78aded
Show file tree
Hide file tree
Showing 10 changed files with 256 additions and 306 deletions.
4 changes: 4 additions & 0 deletions Frends.SFTP.UploadFiles/CHANGELOG.md
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
# Changelog

## [2.7.1] - 2023-07-25
### Added
- Added SFTPClient.OperationTimeout which should cause timeout during operations if the Task becomes stuck.

## [2.7.0] - 2023-06-08
### Added
- [Breaking] Added new parameter for keyboard-interactive authentication where users can add prompts and responses.
Expand Down
2 changes: 1 addition & 1 deletion Frends.SFTP.UploadFiles/Frends.SFTP.UploadFiles.Tests/Lib/UploadFilesTestBase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ public virtual void OneTimeSetup()
Directory = _workDir,
FileName = "SFTPUploadTestFile1.txt",
Action = SourceAction.Error,
Operation = SourceOperation.Nothing,
Operation = SourceOperation.Nothing
};

_destination = new Destination
Expand Down
24 changes: 10 additions & 14 deletions Frends.SFTP.UploadFiles/Frends.SFTP.UploadFiles/Definitions/FileTransporter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ internal FileTransporter(ISFTPLogger logger, BatchContext context, Guid instance
_logger = logger;
_batchContext = context;
_instanceId = instanceId;
_renamingPolicy = new RenamingPolicy(_batchContext.Info.TransferName, _instanceId);
_renamingPolicy = new RenamingPolicy(_batchContext.Info.TransferName, _instanceId, cancellationToken);
_cancellationToken = cancellationToken;

_result = new List<SingleFileTransferResult>();
Expand Down Expand Up @@ -96,7 +96,7 @@ public FileTransferResult Run()
return FormFailedFileTransferResult(userResultMessage);
}

LogSourceSystemInfo(_batchContext, connectionInfo, _logger);
LogDestinationSystemInfo(_batchContext, _logger);

_logger.NotifyInformation(_batchContext, "Negotiation started.");

Expand All @@ -119,6 +119,7 @@ public FileTransferResult Run()

client.ConnectionInfo.Timeout = TimeSpan.FromSeconds(_batchContext.Connection.ConnectionTimeout);
client.KeepAliveInterval = TimeSpan.FromMilliseconds(_batchContext.Connection.KeepAliveInterval);
client.OperationTimeout = TimeSpan.FromSeconds(_batchContext.Connection.ConnectionTimeout);

client.BufferSize = _batchContext.Connection.BufferSize * 1024;

Expand All @@ -142,7 +143,7 @@ public FileTransferResult Run()
try
{
SetCurrentState(TransferState.CreateDestinationDirectories, $"Creating destination directory {DestinationDirectoryWithMacrosExtended}.");
CreateDestinationDirectories(client, DestinationDirectoryWithMacrosExtended);
CreateDestinationDirectories(client, DestinationDirectoryWithMacrosExtended, _cancellationToken);
_logger.NotifyInformation(_batchContext, $"DIRECTORY CREATE: Destination directory {DestinationDirectoryWithMacrosExtended} created.");
}
catch (Exception ex)
Expand All @@ -160,7 +161,6 @@ public FileTransferResult Run()
}
}


_batchContext.DestinationFiles = client.ListDirectory(DestinationDirectoryWithMacrosExtended);

foreach (var file in files)
Expand Down Expand Up @@ -402,7 +402,7 @@ private void CheckServerFingerprint(SftpClient client, string expectedServerFing
{
using (SHA256 mySHA256 = SHA256.Create())
{
SHAServerFingerprint = Util.ToHex(mySHA256.ComputeHash(e.HostKey));
SHAServerFingerprint = Util.ToHex(mySHA256.ComputeHash(e.HostKey), _cancellationToken);
}
e.CanTrust = (SHAServerFingerprint == expectedServerFingerprint);
if (!e.CanTrust)
Expand Down Expand Up @@ -472,11 +472,12 @@ private Tuple<List<FileItem>, bool> ListSourceFiles(Source source, CancellationT
return new Tuple<List<FileItem>, bool>(fileItems, true);
}

private static void CreateDestinationDirectories(SftpClient client, string path)
private static void CreateDestinationDirectories(SftpClient client, string path, CancellationToken cancellationToken)
{
// Consistent forward slashes
foreach (string dir in path.Replace(@"\", "/").Split('/'))
{
cancellationToken.ThrowIfCancellationRequested();
if (!string.IsNullOrWhiteSpace(dir))
{
if (!TryToChangeDir(client, dir) && ("/" + dir != client.WorkingDirectory))
Expand Down Expand Up @@ -528,7 +529,7 @@ private FileTransferResult FormResultFromSingleTransferResults(List<SingleFileTr
{
var success = singleResults.All(x => x.Success);
var actionSkipped = success && singleResults.All(x => x.ActionSkipped);
var userResultMessage = GetUserResultMessage(singleResults.ToList());
var userResultMessage = GetUserResultMessage(singleResults.ToList(), _cancellationToken);

_logger.LogBatchFinished(_batchContext, userResultMessage, success, actionSkipped);

Expand All @@ -552,12 +553,7 @@ private FileTransferResult FormResultFromSingleTransferResults(List<SingleFileTr
};
}

/// <summary>
/// Forms the userResultMessage for FileTransferResult object
/// </summary>
/// <param name="results"></param>
/// <returns></returns>
private static string GetUserResultMessage(IList<SingleFileTransferResult> results)
private static string GetUserResultMessage(IList<SingleFileTransferResult> results, CancellationToken cancellationToken)
{
var userResultMessage = string.Empty;

Expand Down Expand Up @@ -635,7 +631,7 @@ private void SetCurrentState(TransferState state, string msg)
_logger.NotifyTrace($"{state}: {msg}");
}

private static void LogSourceSystemInfo(BatchContext context, ConnectionInfo connectionInfo, ISFTPLogger logger)
private static void LogDestinationSystemInfo(BatchContext context, ISFTPLogger logger)
{
logger.NotifyInformation(context, $"Assembly: {Assembly.GetAssembly(typeof(SftpClient)).GetName().Name} {Assembly.GetAssembly(typeof(SftpClient)).GetName().Version}");
var bit = Environment.Is64BitOperatingSystem ? "64-bit" : "32-bit";
Expand Down
30 changes: 17 additions & 13 deletions Frends.SFTP.UploadFiles/Frends.SFTP.UploadFiles/Definitions/RenamingPolicy.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,13 @@ internal class RenamingPolicy
{
private readonly IDictionary<string, Func<string, string>> MacroHandlers;
private readonly IDictionary<string, Func<string, string>> SourceFileNameMacroHandlers;
private CancellationToken CancellationToken;

public RenamingPolicy(string transferName, Guid transferId)
public RenamingPolicy(string transferName, Guid transferId, CancellationToken cancellationToken)
{
MacroHandlers = InitializeMacroHandlers(transferName, transferId);
SourceFileNameMacroHandlers = InitializeSourceFileNameMacroHandlers();
CancellationToken = cancellationToken;
}

public string CreateRemoteFileName(string originalFileName, string remoteFileDefinition)
Expand All @@ -29,8 +31,8 @@ public string CreateRemoteFileName(string originalFileName, string remoteFileDef
if (string.IsNullOrEmpty(remoteFileDefinition)) return originalFileNameWithoutPath;

if (!IsFileMask(remoteFileDefinition) &&
!IsFileMacro(remoteFileDefinition, MacroHandlers) &&
!IsFileMacro(remoteFileDefinition, SourceFileNameMacroHandlers))
!IsFileMacro(remoteFileDefinition, MacroHandlers, CancellationToken) &&
!IsFileMacro(remoteFileDefinition, SourceFileNameMacroHandlers, CancellationToken))
{
// remoteFileDefination does not have macros
var remoteFileName = Path.GetFileName(remoteFileDefinition);
Expand All @@ -56,7 +58,7 @@ public string ExpandDirectoryForMacros(string directory)
return ExpandFileMacros(directory);
}

public string CanonizeAndCheckPath(string path)
public static string CanonizeAndCheckPath(string path)
{
path = path.Replace(Path.DirectorySeparatorChar, '/'); // make all the paths use forward slashes - this should be supported on File, FTP, and SFTP

Expand Down Expand Up @@ -96,7 +98,7 @@ private string ExpandMacrosAndMasks(string originalFilePath, string filePath)
private string ExpandFileMacros(string filePath)
{
string filename = filePath;
if (IsFileMacro(filename, MacroHandlers))
if (IsFileMacro(filename, MacroHandlers, CancellationToken))
filename = ReplaceMacro(filename);

return filename;
Expand All @@ -105,7 +107,7 @@ private string ExpandFileMacros(string filePath)
private string ExpandSourceFileNameMacros(string filePath, string originalFile)
{
string filename = filePath;
if (IsFileMacro(filename, SourceFileNameMacroHandlers))
if (IsFileMacro(filename, SourceFileNameMacroHandlers, CancellationToken))
filename = ReplaceSourceFileMacro(filename, originalFile);

return filename;
Expand All @@ -130,19 +132,20 @@ private static string NameByMask(string filename, string mask)
if (i >= 0)
{
string tmp = mask.Substring(0, i);
return String.Concat(tmp + filename + mask.Substring(i + 1, (mask.Length - (i + 1))));
return string.Concat(tmp + filename + mask.Substring(i + 1, (mask.Length - (i + 1))));
}

//Not an mask return mask.
return mask;
}

private static bool IsFileMacro(string s, IDictionary<string, Func<string, string>> macroDictionary)
private static bool IsFileMacro(string s, IDictionary<string, Func<string, string>> macroDictionary, CancellationToken cancellationToken)
{
if (s == null) return false;

foreach (var key in macroDictionary.Keys)
{
cancellationToken.ThrowIfCancellationRequested();
if (s.ToUpperInvariant().Contains(key.ToUpperInvariant())) return true;
}

Expand All @@ -163,7 +166,7 @@ private static IDictionary<string, Func<string, string>> InitializeSourceFileNam
return new Dictionary<string, Func<string, string>>
{
{"%SourceFileName%", Path.GetFileNameWithoutExtension},
{"%SourceFileExtension%", (originalFile) => Path.HasExtension(originalFile) ? Path.GetExtension(originalFile) : String.Empty},
{"%SourceFileExtension%", (originalFile) => Path.HasExtension(originalFile) ? Path.GetExtension(originalFile) : string.Empty},
};
}

Expand All @@ -184,26 +187,27 @@ private static IDictionary<string, Func<string, string>> InitializeMacroHandlers
{"%Second%", (s) => DateTime.Now.ToString("ss")},
{"%Millisecond%", (s) => DateTime.Now.ToString("fff")},
{"%Guid%", (s) => Guid.NewGuid().ToString()},
{"%TransferName%", (s) => !String.IsNullOrEmpty(transferName) ? transferName : String.Empty},
{"%TransferName%", (s) => !string.IsNullOrEmpty(transferName) ? transferName : string.Empty},
{"%TransferId%", (s) => transferId.ToString().ToUpper()},
{"%WeekDay%", (s) => (DateTime.Now.DayOfWeek > 0 ? (int)DateTime.Now.DayOfWeek : 7).ToString()}
};
}

private string ReplaceSourceFileMacro(string fileDefinition, string originalFile)
{
return ExpandMacrosFromDictionary(fileDefinition, SourceFileNameMacroHandlers, originalFile); ;
return ExpandMacrosFromDictionary(fileDefinition, SourceFileNameMacroHandlers, originalFile, CancellationToken); ;
}

private string ReplaceMacro(string fileDefinition)
{
return ExpandMacrosFromDictionary(fileDefinition, MacroHandlers, "");
return ExpandMacrosFromDictionary(fileDefinition, MacroHandlers, "", CancellationToken);
}

private static string ExpandMacrosFromDictionary(string fileDefinition, IDictionary<string, Func<string, string>> macroHandlers, string originalFile)
private static string ExpandMacrosFromDictionary(string fileDefinition, IDictionary<string, Func<string, string>> macroHandlers, string originalFile, CancellationToken cancellationToken)
{
foreach (var macroHandler in macroHandlers)
{
cancellationToken.ThrowIfCancellationRequested();
fileDefinition = Regex.Replace(fileDefinition, Regex.Escape(macroHandler.Key), macroHandler.Value.Invoke(originalFile), RegexOptions.IgnoreCase);
}

Expand Down
Loading

0 comments on commit d78aded

Please sign in to comment.