Skip to content

Commit

Permalink
trigger scrape
Browse files Browse the repository at this point in the history
  • Loading branch information
@mlapaglia
mlapaglia committed Jun 22, 2021
1 parent 5660c89 commit 161f009
Show file tree
Hide file tree
Showing 14 changed files with 98 additions and 423 deletions.
7 changes: 5 additions & 2 deletions ...rWebhookProcessor/ClientApp/src/app/settings/openalpr-agent/openalpr-agent.component.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,5 +67,8 @@
style="cursor:default" matSuffix>help_center</mat-icon>
<input type="number" matInput [(ngModel)]="agent.timezoneOffset">
</mat-form-field>
<button mat-raised-button color="primary" fxFlex="80" [disabled]="isSaving" (click)="saveAgent()">Save</button>
</div>
</div>
<div>
<button mat-raised-button color="primary" [disabled]="isSaving" style="margin: 15px;" (click)="saveAgent()">Save</button>
<button mat-raised-button color="primary" [disabled]="isHydrating" (click)="scrapeAgent()">Start Scrape</button>
</div>
19 changes: 16 additions & 3 deletions ...lprWebhookProcessor/ClientApp/src/app/settings/openalpr-agent/openalpr-agent.component.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import { Component, Input, OnInit } from '@angular/core';
import { Component, OnInit } from '@angular/core';
import { SnackbarService } from '@app/snackbar/snackbar.service';
import { SnackBarType } from '@app/snackbar/snackbartype';
import { SettingsService } from '../settings.service';
import { Agent } from './agent';

Expand All @@ -10,8 +12,10 @@ import { Agent } from './agent';
export class OpenalprAgentComponent implements OnInit {
public agent: Agent;
public isSaving: boolean = false;

constructor(private settingsService: SettingsService) { }
public isHydrating: boolean = false;
constructor(
private settingsService: SettingsService,
private snackBarService: SnackbarService) { }

ngOnInit(): void {
this.getAgent();
Expand All @@ -26,6 +30,15 @@ export class OpenalprAgentComponent implements OnInit {
});
}

public scrapeAgent() {
this.isHydrating = true;

this.settingsService.startAgentScrape().subscribe(_ => {
this.isHydrating = false;
this.snackBarService.create("Agent Scraping has begun, check system logs for progress", SnackBarType.Info);
});
}

private getAgent() {
this.settingsService.getAgent().subscribe(result => {
this.agent = result;
Expand Down
4 changes: 4 additions & 0 deletions OpenAlprWebhookProcessor/ClientApp/src/app/settings/settings.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,10 @@ export class SettingsService {
return this.http.get<Agent>(`/settings/agent`);
}

startAgentScrape(): Observable<any> {
return this.http.post(`/settings/agent/scrape`, null);
}

getIgnores(): Observable<Ignore[]> {
return this.http.get<Ignore[]>(`/settings/ignores`);
}
Expand Down
5 changes: 5 additions & 0 deletions OpenAlprWebhookProcessor/ClientApp/src/app/signalr/signalr.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,11 @@ export class SignalrService {
this.snackbarService.create(`Connection lost`, SnackBarType.Disconnected);
this.triggerConnectionStatusChange(false);
});

this.hubConnection.on('ScrapeFinished', _ => {
console.log("Scrape finished");
this.snackbarService.create(`Scrape finished!`, SnackBarType.Info);
});
}

public stopConnection() {
Expand Down
26 changes: 0 additions & 26 deletions OpenAlprWebhookProcessor/Hydration/HydrationController.cs
View file

This file was deleted.

231 changes: 29 additions & 202 deletions OpenAlprWebhookProcessor/Hydration/HydrationService.cs
View file
Original file line number Diff line number Diff line change
@@ -1,41 +1,44 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using OpenAlprWebhookProcessor.Data;
using OpenAlprWebhookProcessor.Hydrator.OpenAlprSearch;
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Text.Json;
using Microsoft.Extensions.Hosting;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
using OpenAlprWebhookProcessor.Utilities;
using System.Net;
using System.Collections.Concurrent;
using OpenAlprWebhookProcessor.WebhookProcessor.OpenAlprAgentScraper;
using System;
using Microsoft.Extensions.DependencyInjection;
using OpenAlprWebhookProcessor.Alerts;
using OpenAlprWebhookProcessor.ProcessorHub;
using Microsoft.AspNetCore.SignalR;

namespace OpenAlprWebhookProcessor.Hydrator
{
public class HydrationService : IHostedService
{
private readonly BlockingCollection<string> _hydrationRequestToProcess;

private readonly CancellationTokenSource _cancellationTokenSource;

private Uri _openAlprServerUrl;
private readonly ILogger _logger;

private readonly IServiceScopeFactory _scopeFactory;
private readonly IServiceProvider _serviceProvider;

private readonly ILogger _logger;
private readonly IHubContext<ProcessorHub.ProcessorHub, IProcessorHub> _processorHub;

public HydrationService(
IServiceScopeFactory scopeFactory,
ILogger<HydrationService> logger)
IServiceProvider serviceProvider,
ILogger<HydrationService> logger,
IHubContext<ProcessorHub.ProcessorHub, IProcessorHub> processorHub)
{
_cancellationTokenSource = new CancellationTokenSource();
_scopeFactory = scopeFactory;
_serviceProvider = serviceProvider;
_logger = logger;
_processorHub = processorHub;
_hydrationRequestToProcess = new BlockingCollection<string>();
}

public Task StartAsync(CancellationToken cancellationToken)
{
Task.Run(() => StartHydrationAsync(), cancellationToken);
return Task.CompletedTask;
}

Expand All @@ -45,202 +48,26 @@ public Task StopAsync(CancellationToken cancellationToken)
return Task.CompletedTask;
}

public void StartHydration()
public void StartHydration(string request)
{
Task.Run(() => StartHydrationAsync());
_hydrationRequestToProcess.Add(request);
}

private async Task StartHydrationAsync()
{
var httpClient = new HttpClient();

var startDate = new DateTimeOffset(2019, 5, 5, 0, 0, 0, TimeSpan.Zero);

var stopDate = DateTimeOffset.UtcNow;

var firstRecordDate = await FindEarliestPlateGroupAsync(
startDate,
httpClient);

if (firstRecordDate == null)
{
return;
}

using (var scope = _scopeFactory.CreateScope())
{
var processorContext = scope.ServiceProvider.GetRequiredService<ProcessorContext>();

var agent = await processorContext.Agents.FirstOrDefaultAsync();

_openAlprServerUrl = new Uri(
Flurl.Url.Combine(
agent.OpenAlprWebServerUrl.ToString(),
"/api/search/plate",
$"?api_key={agent.OpenAlprWebServerApiKey}"));
}

try
{
var responses = new List<Response>();

while (firstRecordDate <= stopDate)
{
var apiResults = await GetOpenAlprPlateGroupsFromApiAsync(
httpClient,
firstRecordDate.Value,
firstRecordDate.Value.AddDays(1));

responses.AddRange(apiResults);

_logger.LogInformation($"pulling plates from: {firstRecordDate.Value:s} to {firstRecordDate.Value.AddDays(1):s}, found {apiResults.Count} plates");

firstRecordDate = firstRecordDate.Value.AddDays(1);
}

var plateGroups = new List<PlateGroup>();

foreach (var response in responses)
{
plateGroups.Add(MapResponseToPlate(response));
}

using (var scope = _scopeFactory.CreateScope())
{
var processorContext = scope.ServiceProvider.GetRequiredService<ProcessorContext>();

_logger.LogInformation($"truncating plates table");

await processorContext.Database.ExecuteSqlRawAsync("DELETE FROM PlateGroups;");

_logger.LogInformation($"populating plates table");

processorContext.AddRange(plateGroups);
await processorContext.SaveChangesAsync();
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Unable to map hydration results");
throw;
}
}

private static PlateGroup MapResponseToPlate(Response apiResponse)
{
var fields = apiResponse.Fields;

var plateGroup = new PlateGroup()
{
Confidence = double.Parse(fields.BestConfidence),
Direction = fields.DirectionOfTravelDegrees,
IsAlert = false,
BestNumber = fields.BestPlate,
OpenAlprCameraId = fields.CameraId,
OpenAlprProcessingTimeMs = double.Parse(fields.ProcessingTimeMs),
OpenAlprUuid = fields.BestUuid,
PlateCoordinates = VehicleUtilities.FormatLicensePlateImageCoordinates(
new List<int>()
{
fields.PlateX1,
fields.PlateX2,
fields.PlateX3,
fields.PlateX4,
},
new List<int>()
{
fields.PlateY1,
fields.PlateY2,
fields.PlateY3,
fields.PlateY4,
}),
ReceivedOnEpoch = DateTimeOffset.Parse(fields.EpochTimeStart).ToUnixTimeMilliseconds(),
VehicleRegion = fields.Region,
VehicleColor = fields.VehicleColor,
VehicleMake = fields.VehicleMake,
VehicleMakeModel = fields.VehicleMakeModel,
VehicleType = fields.VehicleBodyType,
};

return plateGroup;
}

private async Task<List<Response>> GetOpenAlprPlateGroupsFromApiAsync(
HttpClient httpClient,
DateTimeOffset dateRangeStart,
DateTimeOffset dateRangeEnd)
{
var requestUrl = Flurl.Url.Combine(
_openAlprServerUrl.ToString(),
$"start={dateRangeStart.ToString("s", System.Globalization.CultureInfo.InvariantCulture)}",
$"end={dateRangeEnd.ToString("s", System.Globalization.CultureInfo.InvariantCulture)}");

var result = await httpClient.GetAsync(requestUrl);
var response = await result.Content.ReadAsStringAsync();

if (!result.IsSuccessStatusCode)
foreach (var _ in _hydrationRequestToProcess.GetConsumingEnumerable(_cancellationTokenSource.Token))
{
_logger.LogError(
"failed to get plate data, request: {0}, response: {1}",
result.RequestMessage,
response);
}

return JsonSerializer.Deserialize<List<Response>>(response);
}
_logger.LogInformation("Starting OpenALPR Agent scrape.");

private async Task<DateTimeOffset?> FindEarliestPlateGroupAsync(
DateTimeOffset dateRangeStart,
HttpClient httpClient)
{
var numberOfResults = 0;
var currentRequestDate = dateRangeStart;

_logger.LogInformation("searching for first license plate");

try
{
while (numberOfResults == 0)
using (var scope = _serviceProvider.CreateScope())
{
_logger.LogInformation($"searching from: {currentRequestDate:s} to {currentRequestDate.AddDays(1):s}");

var requestUrl = Flurl.Url.Combine(
_openAlprServerUrl.ToString(),
$"start={currentRequestDate.ToString("s", System.Globalization.CultureInfo.InvariantCulture)}",
$"end={currentRequestDate.AddDays(1).ToString("s", System.Globalization.CultureInfo.InvariantCulture)}");

var result = await httpClient.GetAsync(requestUrl);
var response = await result.Content.ReadAsStringAsync();

if (!result.IsSuccessStatusCode)
{
if(result.StatusCode == HttpStatusCode.Unauthorized)
{
_logger.LogError("unauthorized API call, do you have a commercial account?");
}
else
{
_logger.LogError(
"failed to get earliest plate date request: {0} response: {1}",
result.RequestMessage,
response);
}

break;
}

numberOfResults = JsonSerializer.Deserialize<List<Response>>(response).Count;

currentRequestDate = currentRequestDate.AddDays(1);
var scraper = scope.ServiceProvider.GetRequiredService<OpenAlprAgentScraper>();
await scraper.ScrapeAgentAsync(_cancellationTokenSource.Token);
}

return currentRequestDate;
}
catch (Exception ex)
{
_logger.LogError(ex, "failed to get earliest plate date");
await _processorHub.Clients.All.ScrapeFinished();

throw;
_logger.LogInformation("Finished OpenALPR Agent scrape.");
}
}
}
Expand Down
Loading

0 comments on commit 161f009

Please sign in to comment.