Implement OCR in needle editor

os-autoinst · May 30, 2023 · abb46b7 · abb46b7
1 parent ebb84fa
commit abb46b7
Show file tree

Hide file tree

Showing 3 changed files with 100 additions and 26 deletions.
diff --git a/assets/javascripts/needleeditor.js b/assets/javascripts/needleeditor.js
@@ -59,6 +59,33 @@ NeedleEditor.prototype.init = function () {
         return;
       }
       a.type = NeedleEditor.nexttype(a.type);
+
+      // Only image based match areas can utilize margin for fuzzy matching
+      var changeMarginButton = document.getElementById('change-margin');
+      // Only OCR areas can be assigned a refstr
+      var refstrTxtArea = document.getElementById('txtarea-refstr');
+      if (a.type == "match") {
+        changeMarginButton.classList.remove('disabled');
+        changeMarginButton.removeAttribute("disabled");
+        refstrTxtArea.value = '';
+        refstrTxtArea.classList.add('disabled');
+        refstrTxtArea.disabled = "disabled";
+      } else if (a.type == "ocr") {
+        refstrTxtArea.value = '';
+        refstrTxtArea.classList.remove('disabled');
+        refstrTxtArea.removeAttribute("disabled");
+      } else {
+        changeMarginButton.classList.add('disabled');
+        changeMarginButton.disabled = "disabled";
+        refstrTxtArea.value = '';
+        refstrTxtArea.classList.add('disabled');
+        refstrTxtArea.disabled = "disabled";
+      }
+
+      // Attributes only apply to some types of areas
+      delete a.refstr;
+      delete a.margin;
+
       shape.fill = NeedleEditor.areacolor(a.type);
       editor.UpdateTextArea();
       cv.redraw();
@@ -104,21 +131,41 @@ NeedleEditor.prototype.init = function () {
     }
     editor.UpdateTextArea();
   };
+  var matchAreaButtons = $('#change-match, #change-margin, #toggle-click-coordinates');
+  var otherAreaButtons = $('#change-match, #toggle-click-coordinates');
+  var refstrTxtArea = $('#txtarea-refstr');
   cv.new_shape_cb = function (x, y) {
     var a = {xpos: x, ypos: y, width: MINSIZE, height: MINSIZE, type: 'match'};
     var shape = NeedleEditor.ShapeFromArea(a);
     cv.addShape(shape);
     editor.needle.area.push(a);
     editor.UpdateTextArea();
+    matchAreaButtons.removeClass('disabled').removeAttr('disabled');
+    refstrTxtArea.val('');
+    refstrTxtArea.addClass('disabled').attr('disabled', 1);
     return shape;
   };
-  var areaSpecificButtons = $('#change-match, #change-margin, #toggle-click-coordinates');
   $(cv).on('shape.selected', function () {
-    areaSpecificButtons.removeClass('disabled').removeAttr('disabled');
-    updateToggleClickCoordinatesButton(editor.currentClickCoordinates());
+    var selection = editor.selection();
+    var a = selection.area;
+    if (a.type == "match") {
+      matchAreaButtons.removeClass('disabled').removeAttr('disabled');
+      refstrTxtArea.val('');
+      refstrTxtArea.addClass('disabled').attr('disabled', 1);
+    } else if (a.type == "ocr") {
+      otherAreaButtons.removeClass('disabled').removeAttr('disabled');
+      refstrTxtArea.val(a.refstr);
+      refstrTxtArea.removeClass('disabled').removeAttr('disabled');
+    } else {
+      otherAreaButtons.removeClass('disabled').removeAttr('disabled');
+      refstrTxtArea.val('');
+      refstrTxtArea.addClass('disabled').attr('disabled', 1);
+    }
   });
   $(cv).on('shape.unselected', function () {
-    areaSpecificButtons.addClass('disabled').attr('disabled', 1);
+    matchAreaButtons.addClass('disabled').attr('disabled', 1);
+    refstrTxtArea.val('');
+    refstrTxtArea.addClass('disabled').attr('disabled', 1);
   });
 
   document.getElementById('needleeditor_name').onchange = function () {
@@ -160,6 +207,14 @@ NeedleEditor.prototype.AddTag = function (tag, checked) {
   return input;
 };
 
+NeedleEditor.prototype.updateRefstr = function () {
+  var selection = this.selection();
+  var selectedArea = selection.area;
+  var refstrTxtArea = document.getElementById('txtarea-refstr');
+  selectedArea.refstr = refstrTxtArea.value;
+  this.UpdateTextArea();
+};
+
 NeedleEditor.nexttype = function (type) {
   if (type == 'match') {
     return 'exclude';
@@ -457,6 +512,9 @@ function setMatch() {
   nEditor.setMatch($('#match').val());
 }
 
+function updateRefstr() {
+  nEditor.updateRefstr();
+}
 function toggleClickCoordinates() {
   updateToggleClickCoordinatesButton(nEditor.toggleClickCoordinates());
 }

diff --git a/lib/OpenQA/Task/Needle/Save.pm b/lib/OpenQA/Task/Needle/Save.pm
@@ -31,8 +31,6 @@ sub _json_validation {
     if (!exists $djson->{tags} || !exists $djson->{tags}[0]) {
         die 'no tag defined';
     }
-    my @not_ocr_area = grep { $_->{type} ne 'ocr' } @{$djson->{area}};
-    die 'Cannot create a needle with only OCR areas' if scalar(@not_ocr_area) == 0;
 
     my $areas = $djson->{area};
     foreach my $area (@$areas) {
@@ -41,6 +39,11 @@ sub _json_validation {
         die 'area without type' unless exists $area->{type};
         die 'area without height' unless exists $area->{height};
         die 'area without width' unless exists $area->{width};
+        if ($area->{type} eq 'ocr') {
+          die 'OCR area without refstr' unless exists $area->{refstr};
+          die 'refstr is an empty string' if ($area->{refstr} eq '');
+          die 'refstr contains placeholder char §' if ($area->{refstr} =~ qr/§/);
+        }
     }
     return $djson;
 }
@@ -78,21 +81,26 @@ sub _save_needle {
         return $minion_job->finish({error => "<strong>Failed to validate $needlename.</strong><br>$error"});
     }
 
-    # determine imagepath
+    my @match_areas = grep { $_->{type} eq 'match' } @{$json_data->{area}};
+    my $no_match_areas = not scalar(@match_areas);
+
     my $imagepath;
-    if ($imagedir) {
-        $imagepath = join('/', $imagedir, $imagename);
-    }
-    elsif ($imagedistri) {
-        $imagepath = join('/', needledir($imagedistri, $imageversion), $imagename);
-    }
-    else {
-        $imagepath = join('/', $openqa_job->result_dir(), $imagename);
-    }
-    if (!-f $imagepath) {
-        my $error = "Image $imagename could not be found!";
-        $app->log->error("Failed to save needle: $error");
-        return $minion_job->fail({error => "<strong>Failed to save $needlename.</strong><br>$error"});
+    unless ($no_match_areas) {
+        # determine imagepath
+        if ($imagedir) {
+            $imagepath = join('/', $imagedir, $imagename);
+        }
+        elsif ($imagedistri) {
+            $imagepath = join('/', needledir($imagedistri, $imageversion), $imagename);
+        }
+        else {
+            $imagepath = join('/', $openqa_job->result_dir(), $imagename);
+        }
+        if (!-f $imagepath) {
+            my $error = "Image $imagename could not be found!";
+            $app->log->error("Failed to save needle: $error");
+            return $minion_job->fail({error => "<strong>Failed to save $needlename.</strong><br>$error"});
+        }
     }
 
     # check whether needle directory actually exists
@@ -112,17 +120,19 @@ sub _save_needle {
 
     # do not overwrite the exist needle if disallow to overwrite
     my $baseneedle = "$needledir/$needlename";
-    if (-e "$baseneedle.png" && !$args->{overwrite}) {
+    if (-e "$baseneedle.json" && !$args->{overwrite}) {
         #my $returned_data = $self->req->params->to_hash;
         #$returned_data->{requires_overwrite} = 1;
         return $minion_job->finish({requires_overwrite => 1});
     }
 
-    # copy image
     my $success = 1;
-    if (!($imagepath eq "$baseneedle.png") && !copy($imagepath, "$baseneedle.png")) {
-        $app->log->error("Copy $imagepath -> $baseneedle.png failed: $!");
-        $success = 0;
+    unless ($no_match_areas) {
+        # copy image
+        if (!($imagepath eq "$baseneedle.png") && !copy($imagepath, "$baseneedle.png")) {
+            $app->log->error("Copy $imagepath -> $baseneedle.png failed: $!");
+            $success = 0;
+        }
     }
     if ($success) {
         open(my $J, ">", "$baseneedle.json") or $success = 0;
@@ -138,9 +148,11 @@ sub _save_needle {
 
     # commit needle in Git repository
     if ($git->enabled) {
+        my @files_to_be_comm = ["$needlename.json"];
+        push(@files_to_be_comm, "$needlename.png") unless $no_match_areas;
         my $error = $git->commit(
             {
-                add => ["$needlename.json", "$needlename.png"],
+                add => @files_to_be_comm,
                 message => ($commit_message || sprintf("%s for %s", $needlename, $openqa_job->name)),
             });
         if ($error) {

diff --git a/templates/webapi/step/edit.html.ep b/templates/webapi/step/edit.html.ep
@@ -266,6 +266,10 @@
                             <span id="toggle-click-coordinates-verb">Add</span> click coordinates for<br> <code>assert_and_click</code>
                         % end
                     % end
+                    %= tag div => begin
+                        %= tag textarea => (id => 'txtarea-refstr') => (class => 'form-control') => (disabled => undef) => (placeholder => 'OCR Reference Text') => (oninput => 'updateRefstr()') => begin
+                        % end
+                    % end
                 </div>
             </div>
             <div style="margin: 10px auto;">