Merge branch 'next-beta' into feature/blob-open-read

spotflow-io · Aug 16, 2024 · 6170dff · 6170dff
2 parents 83fba11 + 479b2f9
commit 6170dff
Show file tree

Hide file tree

Showing 6 changed files with 48 additions and 29 deletions.
diff --git a/docs/storage.md b/docs/storage.md
@@ -237,19 +237,19 @@ Clients are thread-safe.
 | `Name`              |                         |
 | `Uri`               |                         |
 
-| Method group                 | Note |
-| ---------------------------- | ---- |
-| `Create`                     |      |
-| `CreateIfNotExists`          |      |
-| `DeleteBlob`                 |      |
-| `DeleteBlobIfExists`         |      |
-| `Exists`                     |      |
-| `GetBlobs`                   |      |
-| `GetBlobClient`              |      |
-| `GetBlockBlobClient`         |      |
-| `GetParentBlobServiceClient` |      |
-| `GetProperties`              |      |
-| `UploadBlob`                 |      |
+| Method group                 | Note                                                                                           |
+| ---------------------------- | ---------------------------------------------------------------------------------------------- |
+| `Create`                     |                                                                                                |
+| `CreateIfNotExists`          |                                                                                                |
+| `DeleteBlob`                 |                                                                                                |
+| `DeleteBlobIfExists`         |                                                                                                |
+| `Exists`                     |                                                                                                |
+| `GetBlobClient`              |                                                                                                |
+| `GetBlockBlobClient`         |                                                                                                |
+| `GetBlobs`                   | The `BlobTraits` and `BlobStates` parameters are ignored except `BlobStates.Uncommitted` flag. |
+| `GetParentBlobServiceClient` |                                                                                                |
+| `GetProperties`              |                                                                                                |
+| `UploadBlob`                 |                                                                                                |
 
 | Constructors & factory methods                                          | Note                          |
 | ----------------------------------------------------------------------- | ----------------------------- |
@@ -457,7 +457,7 @@ Following hooks are supported in both `Before` and `After` variants:
             -   `Upload`
             -   `OpenRead`
         -   All `Container` operations
-            -   `Create`
+            -   `Create` / `CreateIfNotExists`
     -   All `Table Service` operations
         -   All `Entity` operations
             -   `Add`

diff --git a/src/Spotflow.InMemory.Azure.Storage/Blobs/InMemoryBlobContainerClient.cs b/src/Spotflow.InMemory.Azure.Storage/Blobs/InMemoryBlobContainerClient.cs
@@ -207,7 +207,7 @@ public override AsyncPageable<BlobItem> GetBlobsAsync(
         string? prefix = null,
         CancellationToken cancellationToken = default)
     {
-        var blobs = GetBlobsCore(prefix);
+        var blobs = GetBlobsCore(prefix, states);
         return new InMemoryPageable.YieldingAsync<BlobItem>(blobs, _defaultMaxPageSize);
     }
 
@@ -217,16 +217,16 @@ public override Pageable<BlobItem> GetBlobs(
         string? prefix = null,
         CancellationToken cancellationToken = default)
     {
-        var blobs = GetBlobsCore(prefix);
+        var blobs = GetBlobsCore(prefix, states);
         return new InMemoryPageable.Sync<BlobItem>(blobs, _defaultMaxPageSize);
     }
 
 
-    private IReadOnlyList<BlobItem> GetBlobsCore(string? prefix)
+    private IReadOnlyList<BlobItem> GetBlobsCore(string? prefix, BlobStates? states)
     {
         var container = GetContainer();
 
-        return container.GetBlobs(prefix);
+        return container.GetBlobs(prefix, states);
     }
 
     #endregion

diff --git a/src/Spotflow.InMemory.Azure.Storage/Blobs/Internals/InMemoryBlobContainer.cs b/src/Spotflow.InMemory.Azure.Storage/Blobs/Internals/InMemoryBlobContainer.cs
@@ -30,17 +30,27 @@ public BlobContainerProperties GetProperties()
 
     public override string? ToString() => $"{Service} / {Name}";
 
-    public IReadOnlyList<BlobItem> GetBlobs(string? prefix)
+    public IReadOnlyList<BlobItem> GetBlobs(string? prefix, BlobStates? states)
     {
         lock (_lock)
         {
             return _blobEntries
                 .Values
-                .Where(entry => entry.Blob.Exists)
-                .Where(entry => prefix is null ? true : entry.Blob.Name.StartsWith(prefix))
+                .Where(entry => filter(entry.Blob))
                 .Select(entry => BlobsModelFactory.BlobItem(entry.Blob.Name))
                 .ToList();
         }
+
+        bool filter(InMemoryBlockBlob blob)
+        {
+            var result = true;
+
+            result &= blob.Exists || (states?.HasFlag(BlobStates.Uncommitted) is true && blob.HasUncommittedBlocks);
+            result &= prefix is null || blob.Name.StartsWith(prefix);
+
+            return result;
+        }
+
     }
 
     public AcquiredBlob AcquireBlob(string blobName, CancellationToken cancellationToken)

diff --git a/src/Spotflow.InMemory.Azure.Storage/Blobs/Internals/InMemoryBlockBlob.cs b/src/Spotflow.InMemory.Azure.Storage/Blobs/Internals/InMemoryBlockBlob.cs
@@ -28,6 +28,8 @@ internal class InMemoryBlockBlob(string blobName, InMemoryBlobContainer containe
 
     public bool Exists => _properties is not null;
 
+    public bool HasUncommittedBlocks => _uncommittedBlocks is not null;
+
     public bool TryGetProperties(
         BlobRequestConditions? conditions,
         [NotNullWhen(true)] out BlobProperties? properties,

diff --git a/src/Spotflow.InMemory.Azure/Internals/TaskExtensions.cs b/src/Spotflow.InMemory.Azure/Internals/TaskExtensions.cs
@@ -1,5 +1,3 @@
-using System.Runtime.CompilerServices;
-
 namespace Spotflow.InMemory.Azure.Internals;
 
 internal static class TaskExtensions

diff --git a/tests/Tests/Storage/Blobs/BlobContainerClientTests.cs b/tests/Tests/Storage/Blobs/BlobContainerClientTests.cs
@@ -1,6 +1,7 @@
 using Azure;
 using Azure.Storage.Blobs;
 using Azure.Storage.Blobs.Models;
+using Azure.Storage.Blobs.Specialized;
 
 using Spotflow.InMemory.Azure.Storage;
 using Spotflow.InMemory.Azure.Storage.Blobs;
@@ -142,23 +143,31 @@ public void Exists_For_Existing_Container_Should_Be_True()
 
     [TestMethod]
     [TestCategory(TestCategory.AzureInfra)]
-    public void GetBlobs_Should_Return_Existing_Blobs()
+    [DataRow(10, 1, BlobStates.None, 10)]
+    [DataRow(10, 1, BlobStates.Uncommitted, 11)]
+    public void GetBlobs_Should_Return_Existing_Relevant_Blobs(int commitedCount, int uncommitedCount, BlobStates states, int expectedTotalCount)
     {
         var containerClient = ImplementationProvider.GetBlobContainerClient();
 
         containerClient.CreateIfNotExists();
 
         var blobNamePrefix = Guid.NewGuid().ToString();
 
-        var count = ImplementationProvider.IsAzureConfigAvailable ? 10 : 100_000;
-
-        for (var i = 0; i < count; i++)
+        for (var i = 0; i < commitedCount; i++)
         {
-            var blobClient = containerClient.GetBlobClient($"{blobNamePrefix}_test-blob-{i:D10}");
+            var blobClient = containerClient.GetBlobClient($"{blobNamePrefix}_test-blob-commited-{i:D10}");
             blobClient.Upload(BinaryData.FromString("test"));
         }
 
-        containerClient.GetBlobs(prefix: blobNamePrefix).Should().HaveCount(count);
+        for (var i = 0; i < uncommitedCount; i++)
+        {
+            var blockBlobClient = containerClient.GetBlockBlobClient($"{blobNamePrefix}_test-blob-uncommited-{i:D10}");
+            blockBlobClient.StageBlock(Convert.ToBase64String([1]), BinaryData.FromString("test").ToStream());
+        }
+
+        containerClient.GetBlobs(prefix: blobNamePrefix, states: states)
+            .Should()
+            .HaveCount(expectedTotalCount);
     }
 
     [TestMethod]