Skip to content

Commit

Permalink
Esa dos protection (#120)
Browse files Browse the repository at this point in the history 
* Skipping this test because mockoon doesn't support paging

* Can no longer fetch objects or launches from entities because of DOS protection

* Fixed the mockoon def

* Excluding objects and launches from bulk fetcher
  • Loading branch information
@hughesjs
hughesjs authored Nov 30, 2022
1 parent b661d78 commit 6fa9c35
Show file tree
Hide file tree
Showing 13 changed files with 663 additions and 310 deletions.
703 changes: 473 additions & 230 deletions src/DiscosWebSdk/DiscosWebSdk.MockApi/DISCOSweb-Mock-Definition.json
View file

Large diffs are not rendered by default.

15 changes: 14 additions & 1 deletion src/DiscosWebSdk/DiscosWebSdk.Tests/Clients/DiscosClientTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,12 @@
using System.Reflection;
using System.Threading.Tasks;
using DiscosWebSdk.Clients;
using DiscosWebSdk.Exceptions;
using DiscosWebSdk.Extensions;
using DiscosWebSdk.Interfaces.Clients;
using DiscosWebSdk.Models.ResponseModels;
using DiscosWebSdk.Models.ResponseModels.Entities;
using DiscosWebSdk.Services.Queries;
using DiscosWebSdk.Tests.Misc;
using DiscosWebSdk.Tests.TestDataGenerators;
using Microsoft.Extensions.Logging.Abstractions;
Expand Down Expand Up @@ -40,7 +42,7 @@ public DiscosClientTests()

innerClient.BaseAddress = new(_apiBase);
innerClient.DefaultRequestHeaders.Authorization = new("bearer", Environment.GetEnvironmentVariable("DISCOS_API_KEY"));
_discosClient = new DiscosClient(innerClient, NullLogger<DiscosClient>.Instance);
_discosClient = new DiscosClient(innerClient, NullLogger<DiscosClient>.Instance, new QueryErrataVerificationService());
}

[Theory]
Expand Down Expand Up @@ -151,5 +153,16 @@ public async Task CanGetMultipleOfEveryTypeWithPaginationWithoutQueryParamsNonGe
res.PaginationDetails.PageSize.ShouldBeGreaterThan(1);
res.PaginationDetails.TotalPages.ShouldBeGreaterThan(0);
}

[Theory]
[InlineData("launches")]
[InlineData("objects")]
[InlineData("launches,objects")]
public async Task ThrowsIfEntitiesIncludeLaunchesOrObjects(params string[] includes)
{
string queryString = $"?include={string.Join(',', includes)}";

await Should.ThrowAsync<EsaDosProtectionException>(async () => await _discosClient.GetSingle<Country>("1", queryString));
}

}
5 changes: 3 additions & 2 deletions src/DiscosWebSdk/DiscosWebSdk.Tests/Mapping/JsonApi/DiscosObjects/DiscosObjectClassTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ public override async Task CanGetMultiple()
{
IReadOnlyList<DiscosObjectClass> ocs = await FetchMultiple<DiscosObjectClass>();
ocs.Count.ShouldBeGreaterThan(1);
ocs.First().Name.ShouldBe("Unknown");
ocs.Last().Name.ShouldBe("Other Debris");

ocs.SingleOrDefault(ocs => ocs.Name == "Unknown").ShouldNotBeNull();
ocs.SingleOrDefault(ocs => ocs.Name == "Payload").ShouldNotBeNull();
}
}
8 changes: 4 additions & 4 deletions src/DiscosWebSdk/DiscosWebSdk.Tests/Mapping/JsonApi/Entities/EntityMapperTests.cs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using DiscosWebSdk.Exceptions;
using DiscosWebSdk.Models.ResponseModels.Entities;
using Shouldly;
using Xunit;
Expand Down Expand Up @@ -49,7 +50,7 @@ public override async Task CanGetSingleWithoutLinks()
[Fact]
public override async Task CanGetSingleWithLinks()
{
string[] includes = {"launches", "launchSites", "launchSystems", "objects"};
string[] includes = {"launchSites", "launchSystems"};
string queryString = $"?include={string.Join(',', includes)}";
Country country = await FetchSingle<Country>(_uk.Id, queryString);
country.LaunchSites.Count.ShouldBe(1);
Expand Down Expand Up @@ -91,12 +92,11 @@ public override async Task CanGetSingleWithoutLinks()
[Fact]
public override async Task CanGetSingleWithLinks()
{
string[] includes = {"launches", "launchSites", "launchSystems", "objects", "hostCountry"};
string[] includes = {"launchSites", "launchSystems", "hostCountry"};
string queryString = $"?include={string.Join(',', includes)}";
Organisation org = await FetchSingle<Organisation>(_spaceX.Id, queryString);
org.LaunchSites.Count.ShouldBe(0);
org.Launches.Count.ShouldBeGreaterThan(130); //135 as of 2022-05-28
org.Objects.Count.ShouldBeGreaterThan(150); //159 as of 2022-05-28
org.HostCountry.Name.ShouldBe("United States");
}

[Fact]
Expand Down
1 change: 1 addition & 0 deletions src/DiscosWebSdk/DiscosWebSdk.Tests/Mapping/JsonApi/JsonApiMapperTestBase.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
using DiscosWebSdk.Models.ResponseModels.Orbits;
using DiscosWebSdk.Models.ResponseModels.Propellants;
using DiscosWebSdk.Models.ResponseModels.Reentries;
using DiscosWebSdk.Services.Queries;
using DiscosWebSdk.Tests.Misc;
using Hypermedia.JsonApi.Client;
using JetBrains.Annotations;
Expand Down
17 changes: 17 additions & 0 deletions src/DiscosWebSdk/DiscosWebSdk.Tests/Queries/DiscosQueryBuilderTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using DiscosWebSdk.Exceptions.Queries.Filters.FilterTree;
using DiscosWebSdk.Interfaces.Queries;
using DiscosWebSdk.Models.ResponseModels.DiscosObjects;
using DiscosWebSdk.Models.ResponseModels.Entities;
using DiscosWebSdk.Models.ResponseModels.Propellants;
using DiscosWebSdk.Queries.Builders;
using DiscosWebSdk.Queries.Filters;
Expand Down Expand Up @@ -66,6 +67,22 @@ public void CanAddInclude()
_builder.Build().ShouldBe("?include=reentry");
}

[Theory]
[InlineData(typeof(Entity))]
[InlineData(typeof(Country))]
[InlineData(typeof(Organisation))]
public void ExcludesLaunchesAndObjectsFromAddingAllIncludesOnEntities(Type entityType)
{
Type queryBuilderType = typeof(DiscosQueryBuilder<>).MakeGenericType(entityType);
DiscosQueryBuilder builder = Activator.CreateInstance(queryBuilderType) as DiscosQueryBuilder ?? throw new("Can't construct query builder");
builder.AddAllIncludes(entityType);

string query = builder.Build();
query.ShouldNotContain("launches");
query.ShouldNotContain("objects");
}


[Fact]
public void ThrowsIfIncludeFieldDoesntExist()
{
Expand Down
8 changes: 6 additions & 2 deletions src/DiscosWebSdk/DiscosWebSdk.Tests/Services/BulkFetching/ImmediateBulkFetchServiceTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
using DiscosWebSdk.Models.ResponseModels.Reentries;
using DiscosWebSdk.Queries.Builders;
using DiscosWebSdk.Services.BulkFetching;
using DiscosWebSdk.Services.Queries;
using DiscosWebSdk.Tests.Misc;
using DiscosWebSdk.Tests.TestDataGenerators;
using Microsoft.Extensions.Logging.Abstractions;
Expand Down Expand Up @@ -48,8 +49,11 @@ public ImmediateBulkFetchServiceTests()
innerClient.Timeout = TimeSpan.FromMinutes(20);
innerClient.BaseAddress = new(_apiBase);
innerClient.DefaultRequestHeaders.Authorization = new("bearer", Environment.GetEnvironmentVariable("DISCOS_API_KEY"));
IDiscosClient discosClient = new DiscosClient(innerClient, NullLogger<DiscosClient>.Instance);
_service = new(discosClient, new DiscosQueryBuilder(), NullLogger<ImmediateBulkFetchService>.Instance);
if (NullLogger<DiscosClient>.Instance != null)
{
IDiscosClient discosClient = new DiscosClient(innerClient, NullLogger<DiscosClient>.Instance, new QueryErrataVerificationService());
_service = new(discosClient, new DiscosQueryBuilder(), NullLogger<ImmediateBulkFetchService>.Instance);
}
}

// Added because of a bug where the query builder wasn't being reset
Expand Down
153 changes: 84 additions & 69 deletions src/DiscosWebSdk/DiscosWebSdk/Clients/DiscosClient.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,50 +12,55 @@
using Hypermedia.JsonApi.Client;
using DiscosWebSdk.Extensions;
using DiscosWebSdk.Interfaces.Clients;
using DiscosWebSdk.Interfaces.Queries;
using DiscosWebSdk.Models.Misc;
using DiscosWebSdk.Models.ResponseModels;
using DiscosWebSdk.Services.Queries;
using JetBrains.Annotations;
using Microsoft.Extensions.Logging;

namespace DiscosWebSdk.Clients;

public class DiscosClient : IDiscosClient
{
private const string SingleFetchTemplate = "{0}/{1}{2}";
private const string SingleFetchTemplate = "{0}/{1}{2}";
private const string MultipleFetchTemplate = "{0}{1}";
private readonly HttpClient _client;

private readonly HttpClient _client;
private readonly ILogger<DiscosClient> _logger;

private readonly Dictionary<Type, string> _endpoints = new()
{
{typeof(DiscosObject), "objects"},
{typeof(DiscosObjectClass), "object-classes"},
{typeof(Country), "entities"},
{typeof(Organisation), "entities"},
{typeof(Entity), "entities"},
{typeof(FragmentationEvent), "fragmentations"},
{typeof(Launch), "launches"},
{typeof(LaunchSite), "launch-sites"},
{typeof(LaunchSystem), "launch-systems"},
{typeof(LaunchVehicle), "launch-vehicles"},
{typeof(LaunchVehicleFamily), "launch-vehicles/families"},
{typeof(LaunchVehicleEngine), "launch-vehicles/engines"},
{typeof(LaunchVehicleStage), "launch-vehicles/stages"},
{typeof(InitialOrbitDetails), "initial-orbits"},
{typeof(DestinationOrbitDetails), "destination-orbits"},
{typeof(Propellant), "propellants"},
{typeof(Reentry), "reentries"}
};

private readonly IQueryVerificationService _queryVerificationService;

public DiscosClient(HttpClient client, ILogger<DiscosClient> logger, IQueryVerificationService queryVerificationService)
{
{typeof(DiscosObject), "objects"},
{typeof(DiscosObjectClass), "object-classes"},
{typeof(Country), "entities"},
{typeof(Organisation), "entities"},
{typeof(Entity), "entities"},
{typeof(FragmentationEvent), "fragmentations"},
{typeof(Launch), "launches"},
{typeof(LaunchSite), "launch-sites"},
{typeof(LaunchSystem), "launch-systems"},
{typeof(LaunchVehicle), "launch-vehicles"},
{typeof(LaunchVehicleFamily), "launch-vehicles/families"},
{typeof(LaunchVehicleEngine), "launch-vehicles/engines"},
{typeof(LaunchVehicleStage), "launch-vehicles/stages"},
{typeof(InitialOrbitDetails), "initial-orbits"},
{typeof(DestinationOrbitDetails), "destination-orbits"},
{typeof(Propellant), "propellants"},
{typeof(Reentry), "reentries"}
};

public DiscosClient(HttpClient client, ILogger<DiscosClient> logger)
{
_logger = logger;
_client = client;
_logger = logger;
_queryVerificationService = queryVerificationService;
_client = client;
}


private string GetSingleFetchUrl(string endpoint, string id, string queryString)
private string GetSingleFetchUrl(string endpoint, string id, string queryString)
{
string fetchUrl = string.Format(SingleFetchTemplate, endpoint, id, queryString);
_logger.LogDebug("Fetching from {FetchUrl}", fetchUrl);
Expand All @@ -71,98 +76,108 @@ private string GetMultipleFetchUrl(string endpoint, string queryString)

public async Task<DiscosModelBase?> GetSingle(Type t, string id, string queryString = "")
{
_queryVerificationService.CheckQuery(t, queryString);
_logger.LogInformation("Getting single {Type} with id {Id} and query string {QueryString}", t.Name, id, queryString);
string endpoint = _endpoints[t];
HttpResponseMessage res = await _client.GetAsync(GetSingleFetchUrl(endpoint, id, queryString));
DiscosModelBase? discosModel = await res.Content.ReadAsJsonApiAsync(t, DiscosObjectResolver.CreateResolver());
string endpoint = _endpoints[t];
HttpResponseMessage res = await _client.GetAsync(GetSingleFetchUrl(endpoint, id, queryString));
DiscosModelBase? discosModel = await res.Content.ReadAsJsonApiAsync(t, DiscosObjectResolver.CreateResolver());
_logger.LogInformation("Successfully fetched object with id {Id}", id);
return discosModel;
}

public async Task<T> GetSingle<T>(string id, string queryString = "")
{
_queryVerificationService.CheckQuery<T>(queryString);
_logger.LogInformation("Getting single {Type} with id {Id} and query string {QueryString}", typeof(T).Name, id, queryString);
string endpoint = _endpoints[typeof(T)];
HttpResponseMessage res = await _client.GetAsync(GetSingleFetchUrl(endpoint, id, queryString));
T discosModel = await res.Content.ReadAsJsonApiAsync<T>(DiscosObjectResolver.CreateResolver());
string endpoint = _endpoints[typeof(T)];
HttpResponseMessage res = await _client.GetAsync(GetSingleFetchUrl(endpoint, id, queryString));
T discosModel = await res.Content.ReadAsJsonApiAsync<T>(DiscosObjectResolver.CreateResolver());
_logger.LogInformation("Successfully fetched object with id {Id}", id);
return discosModel;
}


public async Task<IReadOnlyList<T>?> GetMultiple<T>(string queryString = "")
{
_queryVerificationService.CheckQuery<T>(queryString);
_logger.LogInformation("Getting multiple {Type} with query string {QueryString}", typeof(T).Name, queryString);
string endpoint = _endpoints[typeof(T)];
HttpResponseMessage res = await _client.GetAsync(GetMultipleFetchUrl(endpoint, queryString));
List<T>? discosModels = await res.Content.ReadAsJsonApiManyAsync<T>(DiscosObjectResolver.CreateResolver());
string endpoint = _endpoints[typeof(T)];
HttpResponseMessage res = await _client.GetAsync(GetMultipleFetchUrl(endpoint, queryString));
List<T>? discosModels = await res.Content.ReadAsJsonApiManyAsync<T>(DiscosObjectResolver.CreateResolver());
_logger.LogInformation("Successfully fetched {Count} objects", discosModels?.Count ?? 0);
return discosModels;
}

public async Task<ModelsWithPagination<T>> GetMultipleWithPaginationState<T>(string queryString = "") where T: DiscosModelBase
public async Task<ModelsWithPagination<T>> GetMultipleWithPaginationState<T>(string queryString = "") where T : DiscosModelBase
{
_queryVerificationService.CheckQuery<T>(queryString);
_logger.LogInformation("Getting multiple {Type} (+ pagination state) with query string {QueryString}", typeof(T).Name, queryString);
string endpoint = _endpoints[typeof(T)];
HttpResponseMessage res = await _client.GetAsync(GetMultipleFetchUrl(endpoint, queryString));
List<T> discosModels = await res.Content.ReadAsJsonApiManyAsync<T>(DiscosObjectResolver.CreateResolver()) ?? new List<T>();
string endpoint = _endpoints[typeof(T)];
HttpResponseMessage res = await _client.GetAsync(GetMultipleFetchUrl(endpoint, queryString));
List<T> discosModels = await res.Content.ReadAsJsonApiManyAsync<T>(DiscosObjectResolver.CreateResolver()) ?? new List<T>();
_logger.LogInformation("Successfully fetched {Count} objects", discosModels.Count);

PaginationDetails pagDetails = await GetPaginationDetails(res) ?? new()
{
CurrentPage = 1,
TotalPages = 1,
PageSize = discosModels.Count
};



PaginationDetails pagDetails = await GetPaginationDetails(res) ??
new()
{
CurrentPage = 1,
TotalPages = 1,
PageSize = discosModels.Count
};


return new(discosModels, pagDetails);
}

public async Task<IReadOnlyList<DiscosModelBase?>?> GetMultiple(Type t, string queryString = "")
{
_queryVerificationService.CheckQuery(t, queryString);
_logger.LogInformation("Getting multiple {Type} with query string {QueryString}", t.Name, queryString);
string endpoint = _endpoints[t];
HttpResponseMessage res = await _client.GetAsync(GetMultipleFetchUrl(endpoint, queryString));
string endpoint = _endpoints[t];
HttpResponseMessage res = await _client.GetAsync(GetMultipleFetchUrl(endpoint, queryString));
IReadOnlyList<DiscosModelBase?>? discosModels = await res.Content.ReadAsJsonApiManyAsync(t, DiscosObjectResolver.CreateResolver());
_logger.LogInformation("Successfully fetched {Count} objects", discosModels?.Count ?? 0);
return discosModels;
}

public async Task<ModelsWithPagination<DiscosModelBase>> GetMultipleWithPaginationState(Type t, string queryString = "")
{
_queryVerificationService.CheckQuery(t, queryString);
_logger.LogInformation("Getting multiple {Type} (+ pagination state) with query string {QueryString}", t.Name, queryString);
string endpoint = _endpoints[t];
HttpResponseMessage res = await _client.GetAsync(GetMultipleFetchUrl(endpoint, queryString));
IReadOnlyList<DiscosModelBase?> discosModels = await res.Content.ReadAsJsonApiManyAsync(t, DiscosObjectResolver.CreateResolver()) ?? ArraySegment<DiscosModelBase?>.Empty;
string endpoint = _endpoints[t];
HttpResponseMessage res = await _client.GetAsync(GetMultipleFetchUrl(endpoint, queryString));

IReadOnlyList<DiscosModelBase?> discosModels = await res.Content.ReadAsJsonApiManyAsync(t, DiscosObjectResolver.CreateResolver()) ?? ArraySegment<DiscosModelBase?>.Empty;

_logger.LogInformation("Successfully fetched {Count} objects", discosModels.Count);

PaginationDetails pagDetails = await GetPaginationDetails(res) ?? new()
{
CurrentPage = 1,
TotalPages = 1,
PageSize = discosModels.Count
};
return new(discosModels,pagDetails);

PaginationDetails pagDetails = await GetPaginationDetails(res) ??
new()
{
CurrentPage = 1,
TotalPages = 1,
PageSize = discosModels.Count
};
return new(discosModels, pagDetails);
}

private async Task<PaginationDetails?> GetPaginationDetails(HttpResponseMessage res)
{
//TODO - This is inefficient, update Hypermedia lib to allow for deserialisation from string so we don't read twice
using JsonDocument document = await JsonDocument.ParseAsync(await res.Content.ReadAsStreamAsync());
using JsonDocument document = await JsonDocument.ParseAsync(await res.Content.ReadAsStreamAsync());
return document.RootElement.Deserialize<Response>()?.Meta?.PaginationDetails;
}



// Need these to deserialise the pagination data...
[UsedImplicitly]
private class Response
{
[JsonPropertyName("meta")]
public Meta? Meta { get; [UsedImplicitly] init; }
}



[UsedImplicitly]
private class Meta
{
Expand Down
3 changes: 3 additions & 0 deletions src/DiscosWebSdk/DiscosWebSdk/DependencyInjection/DependencyInjectionExtensions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
using DiscosWebSdk.Options;
using DiscosWebSdk.Queries.Builders;
using DiscosWebSdk.Services.BulkFetching;
using DiscosWebSdk.Services.Queries;
using JetBrains.Annotations;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
Expand Down Expand Up @@ -42,6 +43,8 @@ private static IServiceCollection RegisterEverything(this IServiceCollection ser
services.AddTransient<IDiscosQueryBuilder, DiscosQueryBuilder>();
services.AddTransient<IBulkFetchService, ImmediateBulkFetchService>();

services.AddTransient<IQueryVerificationService, QueryErrataVerificationService>();

if (usePolly)
{
IEnumerable<TimeSpan> spans = retrySpans ?? DefaultRetrySpans;
Expand Down
Loading

0 comments on commit 6fa9c35

Please sign in to comment.