I'd like to elaborate on the accepted answer to this question.

I'm looking at improving the minimal shiny app below (extracted from the accepted answer) with the following features:

• 1) draw the rectangle + a text label. The label es from R (input$foo), e.g., from a dropdown. To avoid the edge cases where the labels fall outside the images, labels should be placed inside their rectangles.
• 2) use a different color for the rectangles and their labels depending on the label
• 3) ability for the user to delete a rectangle by double-clicking inside it. In the case of multiple matches (overlap, nested), the rectangle with the smallest area should be deleted.

Brownie points for 1): the dropdown could appear next to the cursor like is done here (code here). If possible, the dropdown list should be passed from server.R and not be fixed/hardcoded. The reason is that depending on some user input, a different dropdown could be shown. E.g., we might have one dropdown for fruits c('banana','pineapple','grapefruit'), one dropdown for animals c('raccoon','dog','cat'), etc.

# JS and CSS modified from: 
css <- "
    #canvas {
        width:2000px;
        height:2000px;
        border: 10px solid transparent;
    }
    .rectangle {
        border: 5px solid #FFFF00;
        position: absolute;
    }
"

js <- 
"function initDraw(canvas) {
    var mouse = {
        x: 0,
        y: 0,
        startX: 0,
        startY: 0
    };
    function setMousePosition(e) {
        var ev = e || window.event; //Moz || IE
        if (ev.pageX) { //Moz
            mouse.x = ev.pageX + window.pageXOffset;
            mouse.y = ev.pageY + window.pageYOffset;
        } else if (ev.clientX) { //IE
            mouse.x = ev.clientX + document.body.scrollLeft;
            mouse.y = ev.clientY + document.body.scrollTop;
        }
    };

    var element = null;    
    canvas.onmousemove = function (e) {
        setMousePosition(e);
        if (element !== null) {
            element.style.width = Math.abs(mouse.x - mouse.startX) + 'px';
            element.style.height = Math.abs(mouse.y - mouse.startY) + 'px';
            element.style.left = (mouse.x - mouse.startX < 0) ? mouse.x + 'px' : mouse.startX + 'px';
            element.style.top = (mouse.y - mouse.startY < 0) ? mouse.y + 'px' : mouse.startY + 'px';
        }
    }

    canvas.onclick = function (e) {
        if (element !== null) {
           var coord = {
               left: element.style.left,
               top: element.style.top,
               width: element.style.width,
               height: element.style.height
            };
            Shiny.onInputChange('rectCoord', coord);
            element = null;
            canvas.style.cursor = \"default\";
        } else {
            mouse.startX = mouse.x;
            mouse.startY = mouse.y;
            element = document.createElement('div');
            element.className = 'rectangle'
            element.style.left = mouse.x + 'px';
            element.style.top = mouse.y + 'px';
            canvas.appendChild(element);
            canvas.style.cursor = \"crosshair\";
        }
    }
};
$(document).on('shiny:sessioninitialized', function(event) {
    initDraw(document.getElementById('canvas'));
});
"

library(shiny)

ui <- fluidPage(
  tags$head(
      tags$style(css),
      tags$script(HTML(js))
  ),
  fluidRow(
      column(width = 6, 
             # inline is necessary
             # ...otherwise we can draw rectangles over entire fluidRow
             uiOutput("canvas", inline = TRUE)),
      column(
          width = 6,
          verbatimTextOutput("rectCoordOutput")
          )
  )
)

server <- function(input, output, session) {
    output$canvas <- renderUI({
        tags$img(src = ".png")
    })
    output$rectCoordOutput <- renderPrint({
        input$rectCoord
    })

}

shinyApp(ui, server)

I'd like to elaborate on the accepted answer to this question.

I'm looking at improving the minimal shiny app below (extracted from the accepted answer) with the following features:

• 1) draw the rectangle + a text label. The label es from R (input$foo), e.g., from a dropdown. To avoid the edge cases where the labels fall outside the images, labels should be placed inside their rectangles.
• 2) use a different color for the rectangles and their labels depending on the label
• 3) ability for the user to delete a rectangle by double-clicking inside it. In the case of multiple matches (overlap, nested), the rectangle with the smallest area should be deleted.

Brownie points for 1): the dropdown could appear next to the cursor like is done here (code here). If possible, the dropdown list should be passed from server.R and not be fixed/hardcoded. The reason is that depending on some user input, a different dropdown could be shown. E.g., we might have one dropdown for fruits c('banana','pineapple','grapefruit'), one dropdown for animals c('raccoon','dog','cat'), etc.

# JS and CSS modified from: https://stackoverflow./a/17409472/8099834
css <- "
    #canvas {
        width:2000px;
        height:2000px;
        border: 10px solid transparent;
    }
    .rectangle {
        border: 5px solid #FFFF00;
        position: absolute;
    }
"

js <- 
"function initDraw(canvas) {
    var mouse = {
        x: 0,
        y: 0,
        startX: 0,
        startY: 0
    };
    function setMousePosition(e) {
        var ev = e || window.event; //Moz || IE
        if (ev.pageX) { //Moz
            mouse.x = ev.pageX + window.pageXOffset;
            mouse.y = ev.pageY + window.pageYOffset;
        } else if (ev.clientX) { //IE
            mouse.x = ev.clientX + document.body.scrollLeft;
            mouse.y = ev.clientY + document.body.scrollTop;
        }
    };

    var element = null;    
    canvas.onmousemove = function (e) {
        setMousePosition(e);
        if (element !== null) {
            element.style.width = Math.abs(mouse.x - mouse.startX) + 'px';
            element.style.height = Math.abs(mouse.y - mouse.startY) + 'px';
            element.style.left = (mouse.x - mouse.startX < 0) ? mouse.x + 'px' : mouse.startX + 'px';
            element.style.top = (mouse.y - mouse.startY < 0) ? mouse.y + 'px' : mouse.startY + 'px';
        }
    }

    canvas.onclick = function (e) {
        if (element !== null) {
           var coord = {
               left: element.style.left,
               top: element.style.top,
               width: element.style.width,
               height: element.style.height
            };
            Shiny.onInputChange('rectCoord', coord);
            element = null;
            canvas.style.cursor = \"default\";
        } else {
            mouse.startX = mouse.x;
            mouse.startY = mouse.y;
            element = document.createElement('div');
            element.className = 'rectangle'
            element.style.left = mouse.x + 'px';
            element.style.top = mouse.y + 'px';
            canvas.appendChild(element);
            canvas.style.cursor = \"crosshair\";
        }
    }
};
$(document).on('shiny:sessioninitialized', function(event) {
    initDraw(document.getElementById('canvas'));
});
"

library(shiny)

ui <- fluidPage(
  tags$head(
      tags$style(css),
      tags$script(HTML(js))
  ),
  fluidRow(
      column(width = 6, 
             # inline is necessary
             # ...otherwise we can draw rectangles over entire fluidRow
             uiOutput("canvas", inline = TRUE)),
      column(
          width = 6,
          verbatimTextOutput("rectCoordOutput")
          )
  )
)

server <- function(input, output, session) {
    output$canvas <- renderUI({
        tags$img(src = "https://www.r-project/logo/Rlogo.png")
    })
    output$rectCoordOutput <- renderPrint({
        input$rectCoord
    })

}

shinyApp(ui, server)

• javascript
• r
• browser
• shiny

Share Follow edited Jan 29, 2020 at 8:40 Antoine asked Jan 22, 2020 at 12:53 AntoineAntoine 1,74944 gold badges2525 silver badges5454 bronze badges 2

• Would a modified version of the bbox-annotator demo work for you? i.e. use an "x" instead of double clicking to remove a rectangle – Hallie Swan Commented Jan 31, 2020 at 0:34
• Whoah, I had replied to this ment shortly after you posted it 4 days ago, and now I see that it was not saved! Anyways, yes, what you propose would work very well for me, thank you. – Antoine Commented Feb 4, 2020 at 8:15

Add a ment  | 

1 Answer 1

Sorted by: Reset to default 8 +100

This solution uses kyamagu's bbox_annotator and is based on demo.html. I'm not familiar with JS, so it's not the prettiest. Limitations are:

1. Choosing a different image url will remove previous rectangles
2. I edited the JS a bit to change the rectangle/text color, so you won't be able to pull directly from the original repo
3. My changes probably broke input_method = "fixed" and "text", I only tested input_method = "select"

ui.R

# Adapted from https://github./kyamagu/bbox-annotator/
# Edited original JS to add color_list as an option
# ...should be the same length as labels
# ...and controls the color of the rectangle
# ...will probably be broken for input_method = "fixed" or "text"
# Also added color as a value in each rectangle entry
js <- '
    $(document).ready(function() {
       // define options to pass to bounding box constructor
        var options = {
          url: "https://www.r-project/logo/Rlogo.svg",
          input_method: "select", 
          labels: [""],
          color_list:  [""], 
          onchange: function(entries) {
                Shiny.onInputChange("rectCoord", JSON.stringify(entries, null, "  "));
          }
        };

        // Initialize the bounding-box annotator.
        var annotator = new BBoxAnnotator(options);

        // Initialize the reset button.
        $("#reset_button").click(function(e) {
            annotator.clear_all();
        })

        // define function to reset the bbox
        // ...upon choosing new label category or new url
        function reset_bbox(options) {
          document.getElementById("bbox_annotator").setAttribute("style", "display:inline-block");
          $(".image_frame").remove();
          annotator = new BBoxAnnotator(options);
        }

        // update image url from shiny
        Shiny.addCustomMessageHandler("change-img-url", function(url) {
          options.url = url;
          options.width = null;
          options.height = null;
          reset_bbox(options);
        });

        // update colors and categories from shiny
        Shiny.addCustomMessageHandler("update-category-list", function(vals) {
          options.labels = Object.values(vals);
          options.color_list = Object.keys(vals);
          reset_bbox(options);
        });

        // redraw rectangles based on list of entries
        Shiny.addCustomMessageHandler("redraw-rects", function(vals) {
          var arr = JSON.parse(vals);
          arr.forEach(function(rect){
             annotator.add_entry(rect);
          });
          if (annotator.onchange) {
             annotator.onchange(annotator.entries);
          }
        }); 
    });
'

ui <- fluidPage(
    tags$head(tags$script(HTML(js)),
              tags$head(
                  tags$script(src = "bbox_annotation.js")
              )),
    titlePanel("Bounding box annotator demo"),
    sidebarLayout(
        sidebarPanel(
            selectInput(
                "img_url",
                "URLs",
                c(
                    "https://www.r-project/logo/Rlogo.svg",
                    "https://github.githubassets./images/modules/logos_page/GitHub-Mark.png"
                )
            ),
            selectInput("category_type", "Label Category", c("animals", "fruits")),
            div(HTML(
                '<input id="reset_button" type="reset" />'
            )),
            HTML(
                '<input id="annotation_data" name="annotation_data" type="hidden" />'
            ),
            hr(),
            h4("Entries"),
            verbatimTextOutput("rectCoordOutput")
        ),
        mainPanel(div(id = "bbox_annotator", style = "display:inline-block"))
    )
)

server.R

server <- function(input, output, session) {
    # user choices
    output$rectCoordOutput <- renderPrint({
        if(!is.null(input$rectCoord)) {
            as.data.frame(jsonlite::fromJSON(input$rectCoord))
        }
    })
    # send chosen URL from shiny to JS
    observeEvent(input$img_url, {
        session$sendCustomMessage("change-img-url", input$img_url)
    })
    # send chosen category list from shiny to JS
    observeEvent(input$category_type, {
        vals <- switch(input$category_type, 
               fruits = list("yellow" = "banana", 
                          "orange" = "pineapple",
                          "pink" = "grapefruit"),
               animals = list("grey" = "raccoon",
                           "brown" = "dog",
                           "tan" = "cat")
               )
        # update category list
        session$sendCustomMessage("update-category-list", vals)
        # redraw rectangles
        session$sendCustomMessage("redraw-rects", input$rectCoord)
    })
}

www/bbox_annotation.js

// Generated by CoffeeScript 2.5.0
(function() {
  // https://github./kyamagu/bbox-annotator/blob/master/bbox_annotator.coffee
  // Use coffee-script piler to obtain a javascript file.

  //    coffee -c bbox_annotator.coffee

  // See http://coffeescript/

  // BBox selection window.
  var BBoxSelector;

  BBoxSelector = class BBoxSelector {
    // Initializes selector in the image frame.
    constructor(image_frame, options) {
      if (options == null) {
        options = {};
      }
      options.input_method || (options.input_method = "text");
      this.image_frame = image_frame;
      this.border_width = options.border_width || 2;
      this.selector = $('<div class="bbox_selector"></div>');
      this.selector.css({
        // rectangle color when dragging
        "border": this.border_width + "px dotted rgb(127,255,127)",
        "position": "absolute"
      });
      this.image_frame.append(this.selector);
      this.selector.css({
        "border-width": this.border_width
      });
      this.selector.hide();
      this.create_label_box(options);
    }

    // Initializes a label input box.
    create_label_box(options) {
      var i, label, len, ref;
      options.labels || (options.labels = ["object"]);
      this.label_box = $('<div class="label_box" style="z-index: 1000"></div>');
      this.label_box.css({
        "position": "absolute"
      });
      this.image_frame.append(this.label_box);
      switch (options.input_method) {
        case 'select':
          if (typeof options.labels === "string") {
            options.labels = [options.labels];
          }
          this.label_input = $('<select class="label_input" name="label"></select>');
          this.label_box.append(this.label_input);
          this.label_input.append($('<option value>choose an item</option>'));
          ref = options.labels;
          for (i = 0, len = ref.length; i < len; i++) {
            label = ref[i];
            this.label_input.append('<option value="' + label + '">' + label + '</option>');
          }
          this.label_input.change(function(e) {
            return this.blur();
          });
          break;
        case 'text':
          if (typeof options.labels === "string") {
            options.labels = [options.labels];
          }
          this.label_input = $('<input class="label_input" name="label" ' + 'type="text" value>');
          this.label_box.append(this.label_input);
          this.label_input.autoplete({
            source: options.labels || [''],
            autoFocus: true
          });
          break;
        case 'fixed':
          if ($.isArray(options.labels)) {
            options.labels = options.labels[0];
          }
          this.label_input = $('<input class="label_input" name="label" type="text">');
          this.label_box.append(this.label_input);
          this.label_input.val(options.labels);
          break;
        default:
          throw 'Invalid label_input parameter: ' + options.input_method;
      }
      return this.label_box.hide();
    }

    // Crop x and y to the image size.
    crop(pageX, pageY) {
      var point;
      return point = {
        x: Math.min(Math.max(Math.round(pageX - this.image_frame.offset().left), 0), Math.round(this.image_frame.width() - 1)),
        y: Math.min(Math.max(Math.round(pageY - this.image_frame.offset().top), 0), Math.round(this.image_frame.height() - 1))
      };
    }

    // When a new selection is made.
    start(pageX, pageY) {
      this.pointer = this.crop(pageX, pageY);
      this.offset = this.pointer;
      this.refresh();
      this.selector.show();
      $('body').css('cursor', 'crosshair');
      return document.onselectstart = function() {
        return false;
      };
    }

    // When a selection updates.
    update_rectangle(pageX, pageY) {
      this.pointer = this.crop(pageX, pageY);
      return this.refresh();
    }

    // When starting to input label.
    input_label(options) {
      $('body').css('cursor', 'default');
      document.onselectstart = function() {
        return true;
      };
      this.label_box.show();
      return this.label_input.focus();
    }

    // Finish and return the annotation.
    finish(options) {
      var data;
      this.label_box.hide();
      this.selector.hide();
      data = this.rectangle();
      data.label = $.trim(this.label_input.val().toLowerCase());
      if (options.input_method !== 'fixed') {
        this.label_input.val('');
      }
      return data;
    }

    // Get a rectangle.
    rectangle() {
      var rect, x1, x2, y1, y2;
      x1 = Math.min(this.offset.x, this.pointer.x);
      y1 = Math.min(this.offset.y, this.pointer.y);
      x2 = Math.max(this.offset.x, this.pointer.x);
      y2 = Math.max(this.offset.y, this.pointer.y);
      return rect = {
        left: x1,
        top: y1,
        width: x2 - x1 + 1,
        height: y2 - y1 + 1
      };
    }

    // Update css of the box.
    refresh() {
      var rect;
      rect = this.rectangle();
      this.selector.css({
        left: (rect.left - this.border_width) + 'px',
        top: (rect.top - this.border_width) + 'px',
        width: rect.width + 'px',
        height: rect.height + 'px'
      });
      return this.label_box.css({
        left: (rect.left - this.border_width) + 'px',
        top: (rect.top + rect.height + this.border_width) + 'px'
      });
    }

    // Return input element.
    get_input_element() {
      return this.label_input;
    }

  };

  // Annotator object definition.
  this.BBoxAnnotator = class BBoxAnnotator {
    // Initialize the annotator layout and events.
    constructor(options) {
      var annotator, image_element;
      annotator = this;
      this.annotator_element = $(options.id || "#bbox_annotator");
      // allow us to access colors and labels in future steps
      this.color_list = options.color_list;
      this.label_list = options.labels;
      this.border_width = options.border_width || 2;
      this.show_label = options.show_label || (options.input_method !== "fixed");
      if (options.multiple != null) {
        this.multiple = options.multiple;
      } else {
        this.multiple = true;
      }
      this.image_frame = $('<div class="image_frame"></div>');
      this.annotator_element.append(this.image_frame);
      if (options.guide) {
        annotator.initialize_guide(options.guide);
      }
      image_element = new Image();
      image_element.src = options.url;
      image_element.onload = function() {
        options.width || (options.width = image_element.width);
        options.height || (options.height = image_element.height);
        annotator.annotator_element.css({
          "width": (options.width + annotator.border_width) + 'px',
          "height": (options.height + annotator.border_width) + 'px',
          "padding-left": (annotator.border_width / 2) + 'px',
          "padding-top": (annotator.border_width / 2) + 'px',
          "cursor": "crosshair",
          "overflow": "hidden"
        });
        annotator.image_frame.css({
          "background-image": "url('" + image_element.src + "')",
          "width": options.width + "px",
          "height": options.height + "px",
          "position": "relative"
        });
        annotator.selector = new BBoxSelector(annotator.image_frame, options);
        return annotator.initialize_events(options);
      };
      image_element.onerror = function() {
        return annotator.annotator_element.text("Invalid image URL: " + options.url);
      };
      this.entries = [];
      this.onchange = options.onchange;
    }

    // Initialize events.
    initialize_events(options) {
      var annotator, selector, status;
      status = 'free';
      this.hit_menuitem = false;
      annotator = this;
      selector = annotator.selector;
      this.annotator_element.mousedown(function(e) {
        if (!annotator.hit_menuitem) {
          switch (status) {
            case 'free':
            case 'input':
              if (status === 'input') {
                selector.get_input_element().blur();
              }
              if (e.which === 1) { // left button
                selector.start(e.pageX, e.pageY);
                status = 'hold';
              }
          }
        }
        annotator.hit_menuitem = false;
        return true;
      });
      $(window).mousemove(function(e) {
        var offset;
        switch (status) {
          case 'hold':
            selector.update_rectangle(e.pageX, e.pageY);
        }
        if (annotator.guide_h) {
          offset = annotator.image_frame.offset();
          annotator.guide_h.css('top', Math.floor(e.pageY - offset.top) + 'px');
          annotator.guide_v.css('left', Math.floor(e.pageX - offset.left) + 'px');
        }
        return true;
      });
      $(window).mouseup(function(e) {
        switch (status) {
          case 'hold':
            selector.update_rectangle(e.pageX, e.pageY);
            selector.input_label(options);
            status = 'input';
            if (options.input_method === 'fixed') {
              selector.get_input_element().blur();
            }
        }
        return true;
      });
      selector.get_input_element().blur(function(e) {
        var data;
        switch (status) {
          case 'input':
            data = selector.finish(options);
            if (data.label) {
              // store color with the entry
              // ...so we can redraw the rectangle upon changing label category
              data.color = annotator.color_list[annotator.label_list.indexOf(data.label)];
              annotator.add_entry(data);
              if (annotator.onchange) {
                annotator.onchange(annotator.entries);
              }
            }
            status = 'free';
        }
        return true;
      });
      selector.get_input_element().keypress(function(e) {
        switch (status) {
          case 'input':
            if (e.which === 13) {
              selector.get_input_element().blur();
            }
        }
        return e.which !== 13;
      });
      selector.get_input_element().mousedown(function(e) {
        return annotator.hit_menuitem = true;
      });
      selector.get_input_element().mousemove(function(e) {
        return annotator.hit_menuitem = true;
      });
      selector.get_input_element().mouseup(function(e) {
        return annotator.hit_menuitem = true;
      });
      return selector.get_input_element().parent().mousedown(function(e) {
        return annotator.hit_menuitem = true;
      });
    }

    // Add a new entry.
    add_entry(entry) {
      var annotator, box_element, close_button, text_box;
      if (!this.multiple) {
        this.annotator_element.find(".annotated_bounding_box").detach();
        this.entries.splice(0);
      }
      this.entries.push(entry);
      box_element = $('<div class="annotated_bounding_box"></div>');
      box_element.appendTo(this.image_frame).css({
        // rectangle color -- when stopped dragging
        "border": this.border_width + "px solid " + entry.color,
        "position": "absolute",
        "top": (entry.top - this.border_width) + "px",
        "left": (entry.left - this.border_width) + "px",
        "width": entry.width + "px",
        "height": entry.height + "px",
        // text color when stopped dragging
        "color": entry.color,
        "font-family": "monospace",
        "font-size": "small"
      });
      close_button = $('<div></div>').appendTo(box_element).css({
        "position": "absolute",
        "top": "-8px",
        "right": "-8px",
        "width": "16px",
        "height": "0",
        "padding": "16px 0 0 0",
        "overflow": "hidden",
        "color": "#fff",
        "background-color": "#030",
        "border": "2px solid #fff",
        "-moz-border-radius": "18px",
        "-webkit-border-radius": "18px",
        "border-radius": "18px",
        "cursor": "pointer",
        "-moz-user-select": "none",
        "-webkit-user-select": "none",
        "user-select": "none",
        "text-align": "center"
      });
      $("<div></div>").appendTo(close_button).html('&#215;').css({
        "display": "block",
        "text-align": "center",
        "width": "16px",
        "position": "absolute",
        "top": "-2px",
        "left": "0",
        "font-size": "16px",
        "line-height": "16px",
        "font-family": '"Helvetica Neue", Consolas, Verdana, Tahoma, Calibri, ' + 'Helvetica, Menlo, "Droid Sans", sans-serif'
      });
      text_box = $('<div></div>').appendTo(box_element).css({
        "overflow": "hidden"
      });
      if (this.show_label) {
        text_box.text(entry.label);
      }
      annotator = this;
      box_element.hover((function(e) {
        return close_button.show();
      }), (function(e) {
        return close_button.hide();
      }));
      close_button.mousedown(function(e) {
        return annotator.hit_menuitem = true;
      });
      close_button.click(function(e) {
        var clicked_box, index;
        clicked_box = close_button.parent(".annotated_bounding_box");
        index = clicked_box.prevAll(".annotated_bounding_box").length;
        clicked_box.detach();
        annotator.entries.splice(index, 1);
        return annotator.onchange(annotator.entries);
      });
      return close_button.hide();
    }

    // Clear all entries.
    clear_all(e) {
      this.annotator_element.find(".annotated_bounding_box").detach();
      this.entries.splice(0);
      return this.onchange(this.entries);
    }

    // Add crosshair guide.
    initialize_guide(options) {
      this.guide_h = $('<div class="guide_h"></div>').appendTo(this.image_frame).css({
        "border": "1px dotted " + (options.color || '#000'),
        "height": "0",
        "width": "100%",
        "position": "absolute",
        "top": "0",
        "left": "0"
      });
      return this.guide_v = $('<div class="guide_v"></div>').appendTo(this.image_frame).css({
        "border": "1px dotted " + (options.color || '#000'),
        "height": "100%",
        "width": "0",
        "position": "absolute",
        "top": "0",
        "left": "0"
      });
    }

  };

}).call(this);