From e2862d20abfbe9dd9750e7d61bd736ffa2e8e6c7 Mon Sep 17 00:00:00 2001 From: Tomas Pajurek Date: Wed, 14 Aug 2024 21:08:31 +0200 Subject: [PATCH 1/2] Update storage docs --- docs/storage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/storage.md b/docs/storage.md index 3ae4a41..b593d8e 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -454,7 +454,7 @@ Following hooks are supported in both `Before` and `After` variants: - `Download` - `Upload` - All `Container` operations - - `Create` + - `Create` / `CreateIfNotExists` - All `Table Service` operations - All `Entity` operations - `Add` From 479b2f95be23fc081746177b162b6d17cd926071 Mon Sep 17 00:00:00 2001 From: Tomas Pajurek Date: Fri, 16 Aug 2024 08:38:51 +0200 Subject: [PATCH 2/2] Add support for BlobStates.Uncommitted for InMemoryBlobContainerClient.GetBlobs method --- docs/storage.md | 28 +++++++++---------- .../Blobs/InMemoryBlobContainerClient.cs | 8 +++--- .../Blobs/Internals/InMemoryBlobContainer.cs | 16 +++++++++-- .../Blobs/Internals/InMemoryBlockBlob.cs | 2 ++ .../Internals/TaskExtensions.cs | 2 -- .../Storage/Blobs/BlobContainerClientTests.cs | 21 ++++++++++---- 6 files changed, 48 insertions(+), 29 deletions(-) diff --git a/docs/storage.md b/docs/storage.md index b593d8e..c358334 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -237,19 +237,19 @@ Clients are thread-safe. | `Name` | | | `Uri` | | -| Method group | Note | -| ---------------------------- | ---- | -| `Create` | | -| `CreateIfNotExists` | | -| `DeleteBlob` | | -| `DeleteBlobIfExists` | | -| `Exists` | | -| `GetBlobs` | | -| `GetBlobClient` | | -| `GetBlockBlobClient` | | -| `GetParentBlobServiceClient` | | -| `GetProperties` | | -| `UploadBlob` | | +| Method group | Note | +| ---------------------------- | ---------------------------------------------------------------------------------------------- | +| `Create` | | +| `CreateIfNotExists` | | +| `DeleteBlob` | | +| `DeleteBlobIfExists` | | +| `Exists` | | +| `GetBlobClient` | | +| `GetBlockBlobClient` | | +| `GetBlobs` | The `BlobTraits` and `BlobStates` parameters are ignored except `BlobStates.Uncommitted` flag. | +| `GetParentBlobServiceClient` | | +| `GetProperties` | | +| `UploadBlob` | | | Constructors & factory methods | Note | | ----------------------------------------------------------------------- | ----------------------------- | @@ -463,4 +463,4 @@ Following hooks are supported in both `Before` and `After` variants: - `Create` - `Query` -For details about concept of hooks, please see the [Hooks](./hooks.md) page. \ No newline at end of file +For details about concept of hooks, please see the [Hooks](./hooks.md) page. diff --git a/src/Spotflow.InMemory.Azure.Storage/Blobs/InMemoryBlobContainerClient.cs b/src/Spotflow.InMemory.Azure.Storage/Blobs/InMemoryBlobContainerClient.cs index c249d58..2f7aae5 100644 --- a/src/Spotflow.InMemory.Azure.Storage/Blobs/InMemoryBlobContainerClient.cs +++ b/src/Spotflow.InMemory.Azure.Storage/Blobs/InMemoryBlobContainerClient.cs @@ -207,7 +207,7 @@ public override AsyncPageable GetBlobsAsync( string? prefix = null, CancellationToken cancellationToken = default) { - var blobs = GetBlobsCore(prefix); + var blobs = GetBlobsCore(prefix, states); return new InMemoryPageable.YieldingAsync(blobs, _defaultMaxPageSize); } @@ -217,16 +217,16 @@ public override Pageable GetBlobs( string? prefix = null, CancellationToken cancellationToken = default) { - var blobs = GetBlobsCore(prefix); + var blobs = GetBlobsCore(prefix, states); return new InMemoryPageable.Sync(blobs, _defaultMaxPageSize); } - private IReadOnlyList GetBlobsCore(string? prefix) + private IReadOnlyList GetBlobsCore(string? prefix, BlobStates? states) { var container = GetContainer(); - return container.GetBlobs(prefix); + return container.GetBlobs(prefix, states); } #endregion diff --git a/src/Spotflow.InMemory.Azure.Storage/Blobs/Internals/InMemoryBlobContainer.cs b/src/Spotflow.InMemory.Azure.Storage/Blobs/Internals/InMemoryBlobContainer.cs index 87f22a9..1ccf07e 100644 --- a/src/Spotflow.InMemory.Azure.Storage/Blobs/Internals/InMemoryBlobContainer.cs +++ b/src/Spotflow.InMemory.Azure.Storage/Blobs/Internals/InMemoryBlobContainer.cs @@ -30,17 +30,27 @@ public BlobContainerProperties GetProperties() public override string? ToString() => $"{Service} / {Name}"; - public IReadOnlyList GetBlobs(string? prefix) + public IReadOnlyList GetBlobs(string? prefix, BlobStates? states) { lock (_lock) { return _blobEntries .Values - .Where(entry => entry.Blob.Exists) - .Where(entry => prefix is null ? true : entry.Blob.Name.StartsWith(prefix)) + .Where(entry => filter(entry.Blob)) .Select(entry => BlobsModelFactory.BlobItem(entry.Blob.Name)) .ToList(); } + + bool filter(InMemoryBlockBlob blob) + { + var result = true; + + result &= blob.Exists || (states?.HasFlag(BlobStates.Uncommitted) is true && blob.HasUncommittedBlocks); + result &= prefix is null || blob.Name.StartsWith(prefix); + + return result; + } + } public AcquiredBlob AcquireBlob(string blobName, CancellationToken cancellationToken) diff --git a/src/Spotflow.InMemory.Azure.Storage/Blobs/Internals/InMemoryBlockBlob.cs b/src/Spotflow.InMemory.Azure.Storage/Blobs/Internals/InMemoryBlockBlob.cs index cc8514d..656639b 100644 --- a/src/Spotflow.InMemory.Azure.Storage/Blobs/Internals/InMemoryBlockBlob.cs +++ b/src/Spotflow.InMemory.Azure.Storage/Blobs/Internals/InMemoryBlockBlob.cs @@ -28,6 +28,8 @@ internal class InMemoryBlockBlob(string blobName, InMemoryBlobContainer containe public bool Exists => _properties is not null; + public bool HasUncommittedBlocks => _uncommittedBlocks is not null; + public bool TryGetProperties( BlobRequestConditions? conditions, [NotNullWhen(true)] out BlobProperties? properties, diff --git a/src/Spotflow.InMemory.Azure/Internals/TaskExtensions.cs b/src/Spotflow.InMemory.Azure/Internals/TaskExtensions.cs index 3ef7ace..ba91864 100644 --- a/src/Spotflow.InMemory.Azure/Internals/TaskExtensions.cs +++ b/src/Spotflow.InMemory.Azure/Internals/TaskExtensions.cs @@ -1,5 +1,3 @@ -using System.Runtime.CompilerServices; - namespace Spotflow.InMemory.Azure.Internals; internal static class TaskExtensions diff --git a/tests/Tests/Storage/Blobs/BlobContainerClientTests.cs b/tests/Tests/Storage/Blobs/BlobContainerClientTests.cs index 0b3eb7e..be504e8 100644 --- a/tests/Tests/Storage/Blobs/BlobContainerClientTests.cs +++ b/tests/Tests/Storage/Blobs/BlobContainerClientTests.cs @@ -1,6 +1,7 @@ using Azure; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; +using Azure.Storage.Blobs.Specialized; using Spotflow.InMemory.Azure.Storage; using Spotflow.InMemory.Azure.Storage.Blobs; @@ -142,7 +143,9 @@ public void Exists_For_Existing_Container_Should_Be_True() [TestMethod] [TestCategory(TestCategory.AzureInfra)] - public void GetBlobs_Should_Return_Existing_Blobs() + [DataRow(10, 1, BlobStates.None, 10)] + [DataRow(10, 1, BlobStates.Uncommitted, 11)] + public void GetBlobs_Should_Return_Existing_Relevant_Blobs(int commitedCount, int uncommitedCount, BlobStates states, int expectedTotalCount) { var containerClient = ImplementationProvider.GetBlobContainerClient(); @@ -150,15 +153,21 @@ public void GetBlobs_Should_Return_Existing_Blobs() var blobNamePrefix = Guid.NewGuid().ToString(); - var count = ImplementationProvider.IsAzureConfigAvailable ? 10 : 100_000; - - for (var i = 0; i < count; i++) + for (var i = 0; i < commitedCount; i++) { - var blobClient = containerClient.GetBlobClient($"{blobNamePrefix}_test-blob-{i:D10}"); + var blobClient = containerClient.GetBlobClient($"{blobNamePrefix}_test-blob-commited-{i:D10}"); blobClient.Upload(BinaryData.FromString("test")); } - containerClient.GetBlobs(prefix: blobNamePrefix).Should().HaveCount(count); + for (var i = 0; i < uncommitedCount; i++) + { + var blockBlobClient = containerClient.GetBlockBlobClient($"{blobNamePrefix}_test-blob-uncommited-{i:D10}"); + blockBlobClient.StageBlock(Convert.ToBase64String([1]), BinaryData.FromString("test").ToStream()); + } + + containerClient.GetBlobs(prefix: blobNamePrefix, states: states) + .Should() + .HaveCount(expectedTotalCount); } [TestMethod]