Upload CSV file support

backend/files/system/openai/upload-csv.post.hl

@@ -0,0 +1,64 @@
+
+/*
+ * Stores the specified file as training data for the specified model.
+ *
+ * Contrary to the other upload endpoint, this assumes it's given a CSV file, and
+ * it will also use ALL columns in the file. It will assume the first column is
+ * the prompt though.
+ */
+.arguments
+   type:string
+   file:*
+.type:internal
+.accept:multipart/form-data
+.description:Imports the specified CSV file as training data for the specified model
+
+// Ensures user is authorized to access endpoint.
+auth.ticket.verify:root
+
+// Sanity checking invocation.
+validators.mandatory:x:@.arguments/*/type
+validators.mandatory:x:@.arguments/*/file
+validators.mandatory:x:@.arguments/*/file/*/name
+validators.mandatory:x:@.arguments/*/file/*/stream
+
+// Reading file data from stream.
+io.stream.read:x:@.arguments/*/file/*/stream
+
+// Converting file to lambda object.
+csv2lambda:x:@io.stream.read
+
+// Opening database connection.
+data.connect:[generic|magic]
+
+   // Iterating through each record in file.
+   for-each:x:@csv2lambda/*
+
+      // Creating our completion.
+      .completion:
+      for-each:x:@.dp/#/*/[1,1000]
+         set-value:x:@.completion
+            strings.concat
+               get-value:x:@.completion
+               .:"\r\n"
+               .:"\r\n"
+               get-name:x:@.dp/#
+               .:": "
+               get-value:x:@.dp/#
+      set-value:x:@.completion
+         strings.trim:x:@.completion
+
+      // Importing item into database.
+      data.create
+         table:ml_training_snippets
+         values
+            type:x:@.arguments/*/type
+            prompt:x:@.dp/#/0
+            completion:x:@.completion
+            meta:x:@.arguments/*/file/*/name
+
+// Returning success to caller.
+get-count:x:@csv2lambda/*
+yield
+   result:success
+   count:x:@get-count

frontend/src/app/components/protected/manage/machine-learning/components/machine-learning-import/machine-learning-import.component.html

@@ -19,7 +19,7 @@ <h3 mat-dialog-title class="mb-0">Import <span class="d-md-inline d-none">traini
         <div class="col-12 mb-3 mt-4">
 
           <p class="text-muted text-small">
-            Crawl and scrape the specified website for training data
+            Crawl and scrape the specified URL for training data
           </p>
 
           <mat-form-field class="standalone-field w-100">
@@ -160,6 +160,58 @@ <h3 mat-dialog-title class="mb-0">Import <span class="d-md-inline d-none">traini
 
     </mat-tab>
 
+    <mat-tab>
+
+      <ng-template mat-tab-label>
+        CSV file
+      </ng-template>
+
+      <!-- Upload CVS file -->
+      <div class="row px-3">
+
+        <div class="col-12 mt-4 mb-3">
+
+          <p class="text-muted text-small">
+            Upload CSV file with unspecified columns. The first column is assumed to be the prompt, and all other columns will become part of the completion.
+          </p>
+        </div>
+
+        <div class="col-12 mt-2">
+
+          <div class="drop-container pointer border-color">
+
+            <input
+              type="file"
+              [(ngModel)]="trainingFileModelCsv"
+              multiple
+              class="w-100 h-100 hide pointer"
+              #uploadCsv
+              (change)="getFileCsv($event)"
+              accept=".csv">
+
+            <button
+              mat-button
+              class="w-100 pt-2"
+              (click)="uploadCsv.click()"
+              *ngIf="trainingFileModelCsv === ''"
+              [disabled]="uploading">
+              <i>Drag & drop your CSV file here</i> <br />
+              <span class="text-primary">Choose a file</span>
+            </button>
+
+            <ng-container *ngIf="trainingFileModelCsv !== ''">
+              <h4 class="mb-0 mt-3 text-center">{{getFileName()}}</h4>
+              <p class="text-center mt-3" *ngIf="uploading">Uploading {{uploadIndex + 1}} of {{files.length}}...</p>
+              <p class="text-center mt-3" *ngIf="!uploading">Done!</p>
+            </ng-container>
+
+          </div>
+
+        </div>
+
+      </div>
+    </mat-tab>
+
     <mat-tab>
 
       <ng-template mat-tab-label>
@@ -172,7 +224,7 @@ <h3 mat-dialog-title class="mb-0">Import <span class="d-md-inline d-none">traini
         <div class="col-12 mt-4 mb-3">
 
           <p class="text-muted text-small">
-            Upload XML, JSON, YAML or CSV files as training data
+            Upload XML, JSON, YAML or CSV files as training data. Notice, this will only import two columns from your files, prompt and completion.
           </p>
 
           <div class="row">

frontend/src/app/components/protected/manage/machine-learning/components/machine-learning-import/machine-learning-import.component.ts

@@ -22,6 +22,7 @@ export class MachineLearningImportComponent {
 
   uploading: boolean = false;
   trainingFileModel: string = '';
+  trainingFileModelCsv: string = '';
   url: string = null;
   delay: number = 1;
   max: number = 25;
@@ -95,6 +96,19 @@ export class MachineLearningImportComponent {
     this.uploadCurrentFile();
   }
 
+  getFileCsv(event: any) {
+
+    if (!event || !event.target.files || event.target.files.length === 0) {
+      return;
+    }
+    this.uploading = true;
+    this.uploadIndex = 0;
+    this.uploadCount = 0;
+    this.files = event.target.files;
+    this.generalService.showLoading();
+    this.uploadCurrentFile(true);
+  }
+
   getFileName() {
 
     if (!this.files || this.files.length === 0 || this.uploadIndex >= this.files.length) {
@@ -107,18 +121,22 @@ export class MachineLearningImportComponent {
    * Private helper methods.
    */
 
-  private uploadCurrentFile() {
+  private uploadCurrentFile(csvFile: boolean = false) {
 
     const formData = new FormData();
     formData.append('file', this.files[this.uploadIndex], this.files[this.uploadIndex].name);
     formData.append('type', this.data.type);
-    formData.append('prompt', this.prompt);
-    formData.append('completion', this.completion);
-    if (this.massage && this.massage !== '') {
-      formData.append('massage', this.massage);
+    if (!csvFile) {
+      formData.append('prompt', this.prompt);
+      formData.append('completion', this.completion);
+      if (this.massage && this.massage !== '') {
+        formData.append('massage', this.massage);
+      }
     }
 
-    this.openAIService.uploadTrainingFile(formData).subscribe({
+    var svc = csvFile ? this.openAIService.uploadCsvFile.bind(this.openAIService) : this.openAIService.uploadTrainingFile.bind(this.openAIService);
+
+    svc(formData).subscribe({
       next: (result: any) => {
 
         this.uploadCount += result.count;
@@ -131,21 +149,29 @@ export class MachineLearningImportComponent {
             this.generalService.hideLoading();
             this.generalService.showFeedback(`${this.uploadCount} training snippets successfully imported`, 'successMessage');
             this.uploading = false;
-            this.trainingFileModel = '';
+            if (csvFile) {
+              this.trainingFileModelCsv = '';
+            } else {
+              this.trainingFileModel = '';
+            }
             this.uploadIndex = 0;
             this.files = null;
             this.matDialog.close();
             return;
           }
 
           // More files remaining.
-          this.uploadCurrentFile();
+          this.uploadCurrentFile(csvFile);
         }, 100);
       },
       error: (error: any) => {
 
         this.uploading = false;
-        this.trainingFileModel = '';
+        if (csvFile) {
+          this.trainingFileModelCsv = '';
+        } else {
+          this.trainingFileModel = '';
+        }
         this.generalService.showFeedback(error?.error?.message, 'errorMessage', 'Ok');
         this.generalService.hideLoading();
       }

frontend/src/app/services/openai.service.ts

@@ -122,6 +122,14 @@ export class OpenAIService {
     return this.httpService.post<any>('/magic/system/openai/upload-training-data', data);
   }
 
+  /**
+   * Uploads the specified training data file to the backend.
+   */
+  uploadCsvFile(data: FormData) {
+
+    return this.httpService.post<any>('/magic/system/openai/upload-csv', data);
+  }
+
   /**
    * Uploads training data to OpenAI and starts a new training session.
    */