b

backend/backend.csproj

@@ -26,7 +26,7 @@
 
   <ItemGroup>
     <PackageReference Include="magic.lambda.system" Version="17.2.0" />
-    <PackageReference Include="magic.library" Version="17.3.5" />
+    <PackageReference Include="magic.library" Version="17.3.7" />
     <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="8.0.0" />
   </ItemGroup>
 

backend/files/system/openai/import-page.post.hl

@@ -7,6 +7,9 @@
    type:string
    threshold:int
    summarize:bool
+   images:bool
+   lists:bool
+   code:bool
    feedback-channel:string
 
 // Ensures user is authorized to access endpoint.
@@ -21,6 +24,9 @@ validators.integer:x:@.arguments/*/threshold
    min:25
 validators.default:x:@.arguments
    summarize:bool:true
+   images:bool:true
+   lists:bool:true
+   code:bool:true
 
 // Doing some basic logging.
 log.info:Crawling page generating OpenAI training snippets
@@ -47,6 +53,9 @@ fork
          feedback-channel:x:@.arguments/*/feedback-channel
          threshold:x:@.arguments/*/threshold
          summarize:x:@.arguments/*/summarize
+         lists:x:@.arguments/*/lists
+         images:x:@.arguments/*/images
+         code:x:@.arguments/*/code
 
       /*
        * Crawling is done.

backend/files/system/openai/import-url.post.hl

@@ -11,12 +11,21 @@
    max:int
    threshold:int
    summarize:bool
+   images:bool
+   lists:bool
+   code:bool
    feedback-channel:string
 
 
 // Ensures user is authorized to access endpoint.
 auth.ticket.verify:root
 
+// Applying defaults.
+validators.default:x:@.arguments
+   images:bool:true
+   lists:bool:true
+   code:bool:true
+
 // Prepending scheme unless given.
 if
    not

backend/files/system/openai/magic.startup/crawling/magic.ai.crawl-site-on-thread.hl

@@ -245,6 +245,9 @@ delete from ml_training_snippets where type = @type and uri like @uri;"
                   type:x:@.arguments/*/type
                   threshold:x:@.arguments/*/threshold
                   feedback-channel:x:@.arguments/*/feedback-channel
+                  images:x:@.arguments/*/images
+                  lists:x:@.arguments/*/lists
+                  code:x:@.arguments/*/code
 
                // Verifying we've got more snippets before applying Crawl-Delay
                if

backend/files/system/openai/magic.startup/crawling/magic.ai.html.extract.hl

@@ -14,6 +14,12 @@ slots.create:magic.ai.html.extract
    validators.mandatory:x:@.arguments/*/url
    validators.url:x:@.arguments/*/url
 
+   // Ensuring defaults.
+   validators.default:x:@.arguments
+      images:bool:true
+      lists:bool:true
+      code:bool:true
+
    // Used to return meta information to caller.
    .meta
       main:int:0
@@ -36,6 +42,9 @@ slots.create:magic.ai.html.extract
    // Converting HTML to Markdown.
    html2markdown:x:@.arguments/*/html
       url:x:@.arguments/*/url
+      images:x:@.arguments/*/images
+      lists:x:@.arguments/*/lists
+      code:x:@.arguments/*/code
 
    // Checking if we have any Markdown at all, and if not we return early.
    if
@@ -54,9 +63,9 @@ slots.create:magic.ai.html.extract
 
    // Retrieving title and description from document.
    set-value:x:@.title
-      get-value:x:@html2markdown/*/title
+      get-value:x:@html2markdown/*/title/[0,1]
    set-value:x:@.description
-      get-value:x:@html2markdown/*/description
+      get-value:x:@html2markdown/*/description/[0,1]
 
    // Adding URLs found as we transformed HTML to Markdown.
    add:x:@.urls

backend/files/system/openai/magic.startup/crawling/magic.ai.url.scrape.hl

@@ -9,6 +9,9 @@
  * - [type] - What type to import training snippets into.
  * - [summarize] - If true will summarize all training snippets that are larger than 1,000 tokens.
  * - [headers] - Optional collection of HTTML headers we should associate with HTTP request.
+ * - [images] - If true will import images
+ * - [lists] - If true will import lists
+ * - [code] - If true will import code
  */
 slots.create:magic.ai.url.scrape
 
@@ -17,9 +20,12 @@ slots.create:magic.ai.url.scrape
    validators.mandatory:x:@.arguments/*/type
    validators.url:x:@.arguments/*/url
 
-   // Ensuring [summarize] defaults to true.
+   // Ensuring defaults.
    validators.default:x:@.arguments
       summarize:bool:true
+      images:bool:true
+      lists:bool:true
+      code:bool:true
 
    // Signaling frontend.
    .msg
@@ -54,6 +60,9 @@ slots.create:magic.ai.url.scrape
          html:x:@.html
          url:x:@.arguments/*/url
          feedback-channel:x:@.arguments/*/feedback-channel
+         images:x:@.arguments/*/images
+         lists:x:@.arguments/*/lists
+         code:x:@.arguments/*/code
 
       // URLs we return to caller.
       .urls

backend/slots/Version.cs

@@ -20,7 +20,7 @@ public class Version : ISlot
         /// <param name="input">Parameters passed from signaler</param>
         public void Signal(ISignaler signaler, Node input)
         {
-            input.Value = "v17.3.5";
+            input.Value = "v17.3.7";
         }
     }
 }

frontend/src/app/components/protected/manage/machine-learning/components/machine-learning-edit-training-snippet/machine-learning-edit-training-snippet.component.html

@@ -180,6 +180,7 @@ <h3 mat-dialog-title class="mb-0">
 
       <button
         mat-flat-button
+        color="primary"
         type="submit">
         Save
       </button>

frontend/src/app/components/protected/manage/machine-learning/components/machine-learning-import-feedback/machine-learning-import-feedback.component.ts

@@ -3,13 +3,16 @@
  * Copyright (c) Thomas Hansen - For license inquiries you can contact thomas@ainiro.io.
  */
 
+// Angular and system specific imports.
+import { MAT_DIALOG_DATA } from '@angular/material/dialog';
 import { Component, Inject, OnDestroy, OnInit } from '@angular/core';
-import { MAT_DIALOG_DATA, MatDialogRef } from '@angular/material/dialog';
 import { HttpTransportType, HubConnection, HubConnectionBuilder } from '@aspnet/signalr';
-import { BackendService } from 'src/app/services/backend.service';
+
+// Application specific imports.
 import { ConfigService } from 'src/app/services/config.service';
-import { GeneralService } from 'src/app/services/general.service';
 import { OpenAIService } from 'src/app/services/openai.service';
+import { BackendService } from 'src/app/services/backend.service';
+import { GeneralService } from 'src/app/services/general.service';
 
 /**
  * Helper component to view feedback as we crawl URLs.
@@ -27,7 +30,6 @@ export class MachineLearningImportFeedbackComponent implements OnInit, OnDestroy
   constructor(
     @Inject(MAT_DIALOG_DATA) public data: any,
     private backendService: BackendService,
-    private dialogRef: MatDialogRef<MachineLearningImportFeedbackComponent>,
     private generalService: GeneralService,
     private openAIService: OpenAIService,
     private configService: ConfigService) { }
@@ -91,7 +93,10 @@ export class MachineLearningImportFeedbackComponent implements OnInit, OnDestroy
               this.data.max,
               this.data.threshold,
               this.data.summarize,
-              result.result).subscribe({
+              result.result,
+              this.data.images,
+              this.data.lists,
+              this.data.code).subscribe({
               next: () => {
 
                 this.generalService.hideLoading();
@@ -112,7 +117,10 @@ export class MachineLearningImportFeedbackComponent implements OnInit, OnDestroy
               this.data.url,
               this.data.type,
               50,
-              result.result).subscribe({
+              result.result,
+              this.data.images,
+              this.data.lists,
+              this.data.code).subscribe({
               next: () => {
 
                 this.generalService.hideLoading();

frontend/src/app/components/protected/manage/machine-learning/components/machine-learning-import/machine-learning-import.component.html

@@ -120,6 +120,33 @@ <h3 mat-dialog-title class="mb-0">Import <span class="d-md-inline d-none">traini
 
       <div class="text-end me-2">
 
+        <mat-slide-toggle
+          labelPosition="before"
+          class="me-5"
+          matTooltip="Import lists"
+          [disabled]="uploading"
+          [(ngModel)]="lists">
+          Lists
+        </mat-slide-toggle>
+
+        <mat-slide-toggle
+          labelPosition="before"
+          class="me-5"
+          matTooltip="Import code segments"
+          [disabled]="uploading"
+          [(ngModel)]="code">
+          Code
+        </mat-slide-toggle>
+
+        <mat-slide-toggle
+          labelPosition="before"
+          class="me-5"
+          matTooltip="Import images"
+          [disabled]="uploading"
+          [(ngModel)]="images">
+          Images
+        </mat-slide-toggle>
+
         <mat-slide-toggle
           labelPosition="before"
           class="me-2"

frontend/src/app/components/protected/manage/machine-learning/components/machine-learning-import/machine-learning-import.component.ts

@@ -33,6 +33,9 @@ export class MachineLearningImportComponent {
   uploadCount: number = 0;
   files: FileList = null;
   summarize: boolean = true;
+  images: boolean = true;
+  code: boolean = true;
+  lists: boolean = true;
   massageTemplate: string = null;
   massageTemplates: string[] = [
     'Summarize the following into a desriptive title and content, separated by carrriage return',
@@ -67,7 +70,16 @@ export class MachineLearningImportComponent {
       return;
     }
 
-    this.matDialog.close({ crawl: this.url, delay: this.delay * 1000, max: this.max, threshold: this.threshold, summarize: this.summarize });
+    this.matDialog.close({
+      crawl: this.url,
+      delay: this.delay * 1000,
+      max: this.max,
+      threshold: this.threshold,
+      summarize: this.summarize,
+      images: this.images,
+      code: this.code,
+      lists: this.lists,
+    });
   }
 
   getFile(event: any) {

frontend/src/app/components/protected/manage/machine-learning/components/machine-learning-spice/machine-learning-spice.component.html

@@ -30,6 +30,33 @@ <h3 mat-dialog-title class="mb-0">Spice up '{{data.type}}' model</h3>
 
   <div mat-dialog-actions [align]="'end'">
 
+    <mat-slide-toggle
+      labelPosition="before"
+      class="me-5"
+      matTooltip="Import lists"
+      name="lists"
+      [(ngModel)]="lists">
+      Lists
+    </mat-slide-toggle>
+
+    <mat-slide-toggle
+      labelPosition="before"
+      class="me-5"
+      matTooltip="Import code segments"
+      name="code"
+      [(ngModel)]="code">
+      Code
+    </mat-slide-toggle>
+
+    <mat-slide-toggle
+      labelPosition="before"
+      class="me-5"
+      matTooltip="Import images"
+      name="images"
+      [(ngModel)]="images">
+      Images
+    </mat-slide-toggle>
+
     <button
       mat-button
       type="button"
@@ -40,6 +67,7 @@ <h3 mat-dialog-title class="mb-0">Spice up '{{data.type}}' model</h3>
     <button
       class="pe-3"
       type="submit"
+      color="primary"
       mat-flat-button>
       Scrape
     </button>

frontend/src/app/components/protected/manage/machine-learning/components/machine-learning-spice/machine-learning-spice.component.ts

@@ -23,14 +23,21 @@ export class MachineLearningSpiceComponent {
 
   CommonRegEx = CommonRegEx;
   CommonErrorMessages = CommonErrorMessages;
+  lists?: boolean = true;
+  images?: boolean = true;
+  code?: boolean = true;
 
   constructor(
-    private generalService: GeneralService,
     @Inject(MAT_DIALOG_DATA) public data: any,
     private dialogRef: MatDialogRef<MachineLearningSpiceComponent>) { }
 
   scrape() {
 
-    this.dialogRef.close(this.url);
+    this.dialogRef.close({
+      url: this.url,
+      lists: this.lists,
+      images: this.images,
+      code: this.code,
+    });
   }
 }

frontend/src/app/components/protected/manage/machine-learning/machine-learning-models/machine-learning-models.component.ts

@@ -153,7 +153,11 @@ export class MachineLearningModelsComponent implements OnInit {
         delay?: number,
         max?: number,
         threshold?: number,
-        summarize?: boolean }) => {
+        summarize?: boolean,
+        images?: boolean,
+        code?: boolean,
+        lists?: boolean,
+      }) => {
 
         if (result?.crawl) {
 
@@ -168,7 +172,10 @@ export class MachineLearningModelsComponent implements OnInit {
                 max: result.max,
                 threshold: result.threshold,
                 summarize: result.summarize,
-                mode: 'site'
+                images: result.images ?? true,
+                code: result.code ?? true,
+                lists: result.lists ?? true,
+                mode: 'site',
               }
             });
         }

frontend/src/app/components/protected/manage/machine-learning/machine-learning-training-data/machine-learning-training-data.component.ts

@@ -121,12 +121,15 @@ export class MachineLearningTrainingDataComponent implements OnInit {
               width: '80vw',
               maxWidth: '1280px',
               data: {
-                url: result,
+                url: result.url,
                 type: this.type,
                 delay: result.delay,
                 max: result.max,
                 threshold: result.threshold,
                 summarize: result.summarize,
+                images: result.images,
+                lists: result.lists,
+                code: result.code,
                 mode: 'single-page',
                 callback: () => this.getTrainingData(true)
               }