From 6df7fa849a37d1e12c49f64908587e85fc5f3755 Mon Sep 17 00:00:00 2001 From: Devin Howard Date: Sun, 7 Feb 2016 16:56:03 +0800 Subject: [PATCH 01/13] bare minimum topic import functionality - use by Ctrl+V onto the map canvas itself --- app/assets/javascripts/application.js | 1 + .../javascripts/src/Metamaps.Import.js.erb | 104 ++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 app/assets/javascripts/src/Metamaps.Import.js.erb diff --git a/app/assets/javascripts/application.js b/app/assets/javascripts/application.js index b3b9cdfe..2fd7ab6a 100644 --- a/app/assets/javascripts/application.js +++ b/app/assets/javascripts/application.js @@ -25,6 +25,7 @@ //= require ./src/views/room //= require ./src/JIT //= require ./src/Metamaps +//= require ./src/Metamaps.Import //= require ./src/Metamaps.JIT //= require_directory ./shims //= require_directory ./require diff --git a/app/assets/javascripts/src/Metamaps.Import.js.erb b/app/assets/javascripts/src/Metamaps.Import.js.erb new file mode 100644 index 00000000..d17077ae --- /dev/null +++ b/app/assets/javascripts/src/Metamaps.Import.js.erb @@ -0,0 +1,104 @@ +/* + * Example tab-separated input: + * Some fields will be ignored + * + * id name metacode desc link user.name permission synapses + * 1 topic1 Catalyst admin commons 1->7 + * 2 topic2 Event admin commons + * 5 topic Action admin commons + * 6 topic6 Action admin commons 6->7 + * 7 topic7 Action admin commons 7->8 7<-6 7<-1 + * 8 topic8 Action admin commons 8<-7 + */ + +Metamaps.Import = { + headersWhitelist: [ + 'name', 'metacode', 'desc', 'link', 'permission' + ], + + init: function() { + var self = Metamaps.Import; + $('body').bind('paste', function(e) { + var text = e.originalEvent.clipboardData.getData('text/plain'); + var parsed = self.parseTabbedString(text); + + if (confirm("Are you sure you want to create " + parsed.length + " new topics?")) { + self.importTopics(parsed); + }//if + }); + }, + + importTopics: function(parsedTopics) { + var self = Metamaps.Import; + + var x = -200; + var y = -200; + parsedTopics.forEach(function(topic) { + self.createTopicWithParameters( + topic.name, topic.metacode, topic.permission, + topic.desc, topic.link, x, y + ); + + // update positions of topics + x += 50; + if (x > 200) { + y += 50; + x = -200; + }//if + }); + }, + + parseTabbedString: function(text) { + var self = Metamaps.Import; + + // determine line ending and split lines + var delim = "\n"; + if (text.indexOf("\r\n") !== -1) { + delim = "\r\n"; + }//if + var lines = text.split(delim); + + // get csv-style headers to name the object fields + var headers = lines[0].split(' '); //tab character + + var results = []; + lines.forEach(function(line, index) { + if (index === 0) return; + if (line == "") return; + + var topic = {}; + line.split(" ").forEach(function(field, index) { + if (self.headersWhitelist.indexOf(headers[index]) === -1) return; + topic[headers[index]] = field; + }); + results.push(topic); + }); + return results; + }, + + createTopicWithParameters: function(name, metacode_name, permission, desc, link, xloc, yloc) { + var self = Metamaps.Topic; + + var metacode = Metamaps.Metacodes.where({name: metacode_name})[0] || null; + if (metacode === null) return console.error("metacode not found"); + + var topic = new Metamaps.Backbone.Topic({ + name: name, + metacode_id: metacode.id, + permission: permission || Metamaps.Active.Map.get('permission'), + desc: desc, + link: link + }); + Metamaps.Topics.add(topic); + + var mapping = new Metamaps.Backbone.Mapping({ + xloc: xloc, + yloc: yloc, + mappable_id: topic.cid, + mappable_type: "Topic", + }); + Metamaps.Mappings.add(mapping); + + self.renderTopic(mapping, topic, true, true); // this function also includes the creation of the topic in the database + }, +}; From b47ed7b5b44f3498c04bdfa4952c2819d35d35b9 Mon Sep 17 00:00:00 2001 From: Devin Howard Date: Sun, 7 Feb 2016 17:51:01 +0800 Subject: [PATCH 02/13] don't ask about adding 0 topics --- app/assets/javascripts/src/Metamaps.Import.js.erb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/app/assets/javascripts/src/Metamaps.Import.js.erb b/app/assets/javascripts/src/Metamaps.Import.js.erb index d17077ae..14612247 100644 --- a/app/assets/javascripts/src/Metamaps.Import.js.erb +++ b/app/assets/javascripts/src/Metamaps.Import.js.erb @@ -22,7 +22,9 @@ Metamaps.Import = { var text = e.originalEvent.clipboardData.getData('text/plain'); var parsed = self.parseTabbedString(text); - if (confirm("Are you sure you want to create " + parsed.length + " new topics?")) { + if (parsed.length > 0 && + confirm("Are you sure you want to create " + parsed.length + + " new topics?")) { self.importTopics(parsed); }//if }); @@ -76,7 +78,8 @@ Metamaps.Import = { return results; }, - createTopicWithParameters: function(name, metacode_name, permission, desc, link, xloc, yloc) { + createTopicWithParameters: function(name, metacode_name, permission, desc, + link, xloc, yloc) { var self = Metamaps.Topic; var metacode = Metamaps.Metacodes.where({name: metacode_name})[0] || null; @@ -99,6 +102,7 @@ Metamaps.Import = { }); Metamaps.Mappings.add(mapping); - self.renderTopic(mapping, topic, true, true); // this function also includes the creation of the topic in the database + // this function also includes the creation of the topic in the database + self.renderTopic(mapping, topic, true, true); }, }; From 0c1e12a301a2daea74f346a786f192c60b22ac75 Mon Sep 17 00:00:00 2001 From: Devin Howard Date: Sun, 14 Feb 2016 22:39:29 +0800 Subject: [PATCH 03/13] use state machine to implement smarter topic/synapse import also include better auto-layout of new topics if x/y not specified --- .../javascripts/src/Metamaps.Import.js.erb | 237 ++++++++++++++---- 1 file changed, 194 insertions(+), 43 deletions(-) diff --git a/app/assets/javascripts/src/Metamaps.Import.js.erb b/app/assets/javascripts/src/Metamaps.Import.js.erb index 14612247..b604e970 100644 --- a/app/assets/javascripts/src/Metamaps.Import.js.erb +++ b/app/assets/javascripts/src/Metamaps.Import.js.erb @@ -12,42 +12,34 @@ */ Metamaps.Import = { - headersWhitelist: [ - 'name', 'metacode', 'desc', 'link', 'permission' + topicWhitelist: [ + 'name', 'metacode', 'description', 'link', 'permission' ], + synapseWhitelist: [ + 'desc', 'description', 'category', 'topic1', 'topic2', 'permission' + ], init: function() { var self = Metamaps.Import; $('body').bind('paste', function(e) { var text = e.originalEvent.clipboardData.getData('text/plain'); - var parsed = self.parseTabbedString(text); - if (parsed.length > 0 && - confirm("Are you sure you want to create " + parsed.length + - " new topics?")) { - self.importTopics(parsed); + var results = self.parseTabbedString(text); + var topics = results.topics; + var synapses = results.synapses; + + if (topics.length > 0 || synapses.length > 0) { + if (confirm("Are you sure you want to create " + topics.length + + " new topics and " + synapses.length + " new synapses?")) { + self.importTopics(topics); + self.importSynapses(synapses); + }//if }//if }); }, - importTopics: function(parsedTopics) { - var self = Metamaps.Import; - - var x = -200; - var y = -200; - parsedTopics.forEach(function(topic) { - self.createTopicWithParameters( - topic.name, topic.metacode, topic.permission, - topic.desc, topic.link, x, y - ); - - // update positions of topics - x += 50; - if (x > 200) { - y += 50; - x = -200; - }//if - }); + abort: function(message) { + console.error(message); }, parseTabbedString: function(text) { @@ -58,30 +50,162 @@ Metamaps.Import = { if (text.indexOf("\r\n") !== -1) { delim = "\r\n"; }//if + + var STATES = { + UNKNOWN: 0, + TOPICS_NEED_HEADERS: 1, + SYNAPSES_NEED_HEADERS: 2, + TOPICS: 3, + SYNAPSES: 4, + }; + + // state & lines determine parser behaviour + var state = STATES.UNKNOWN; var lines = text.split(delim); + var results = { topics: [], synapses: [] } + var topicHeaders = []; + var synapseHeaders = []; - // get csv-style headers to name the object fields - var headers = lines[0].split(' '); //tab character - - var results = []; - lines.forEach(function(line, index) { - if (index === 0) return; - if (line == "") return; - - var topic = {}; - line.split(" ").forEach(function(field, index) { - if (self.headersWhitelist.indexOf(headers[index]) === -1) return; - topic[headers[index]] = field; + lines.forEach(function(line_raw, index) { + var line = line_raw.split(' '); // tab character + var noblanks = line.filter(function(elt) { + return elt !== ""; }); - results.push(topic); + switch(state) { + case STATES.UNKNOWN: + if (noblanks.length === 0) { + state = STATES.UNKNOWN; + break; + } else if (noblanks.length === 1 && line[0].toLowerCase() === 'topics') { + state = STATES.TOPICS_NEED_HEADERS; + break; + } else if (noblanks.length === 1 && line[0].toLowerCase() === 'synapses') { + state = STATES.SYNAPSES_NEED_HEADERS; + break; + } + state = STATES.TOPICS_NEED_HEADERS; + // FALL THROUGH - if we're not sure what to do, pretend + // we're on the TOPICS_NEED_HEADERS state and parse some headers + + case STATES.TOPICS_NEED_HEADERS: + if (noblanks.length < 2) { + return self.abort("Not enough topic headers on line " + index); + } + topicHeaders = line.map(function(header, index) { + return header.toLowerCase().replace('description', 'desc'); + }); + state = STATES.TOPICS; + break; + + case STATES.SYNAPSES_NEED_HEADERS: + if (noblanks.length < 2) { + return self.abort("Not enough synapse headers on line " + index); + } + synapseHeaders = line.map(function(header, index) { + return header.toLowerCase().replace('description', 'desc'); + }); + state = STATES.SYNAPSES; + break; + + case STATES.TOPICS: + if (noblanks.length === 0) { + state = STATES.UNKNOWN; + } else if (noblanks.length === 1 && line[0].toLowerCase() === 'topics') { + state = STATES.TOPICS_NEED_HEADERS; + } else if (noblanks.length === 1 && line[0].toLowerCase() === 'synapses') { + state = STATES.SYNAPSES_NEED_HEADERS; + } else { + var topic = {}; + line.forEach(function(field, index) { + var header = topicHeaders[index]; + if (self.topicWhitelist.indexOf(header) === -1) return; + topic[header] = field; + if (header === 'x' || header === 'y') { + topic[header] = parseInt(topic[header]); + }//if + }); + results.topics.push(topic); + } + break; + + case STATES.SYNAPSES: + if (noblanks.length === 0) { + state = STATES.UNKNOWN; + } else if (noblanks.length === 1 && line[0].toLowerCase() === 'topics') { + state = STATES.TOPICS_NEED_HEADERS; + } else if (noblanks.length === 1 && line[0].toLowerCase() === 'synapses') { + state = STATES.SYNAPSES_NEED_HEADERS; + } else { + var synapse = {}; + line.forEach(function(field, index) { + var header = synapseHeaders[index]; + if (self.synapseWhitelist.indexOf(header) === -1) return; + synapse[header] = field; + if (header === 'topic1' || header === 'topic2') { + synapse[header] = parseInt(header); + }//if + }); + results.synapses.push(synapse); + } + break; + + default: + return self.abort("Invalid state while parsing import data. " + + "Check code."); + } }); + return results; }, - createTopicWithParameters: function(name, metacode_name, permission, desc, - link, xloc, yloc) { - var self = Metamaps.Topic; + importTopics: function(parsedTopics) { + var self = Metamaps.Import; + + // up to 25 topics: scale 100 + // up to 81 topics: scale 200 + // up to 169 topics: scale 300 + var scale = Math.floor((Math.sqrt(parsedTopics.length) - 1) / 4) * 100; + if (scale < 100) scale = 100; + var autoX = -scale; + var autoY = -scale; + + parsedTopics.forEach(function(topic) { + var x, y; + if (topic.x && topic.y) { + x = topic.x; + y = topic.y; + } else { + x = autoX; + y = autoY; + autoX += 50; + if (autoX > scale) { + autoY += 50; + autoX = -scale; + } + } + + self.createTopicWithParameters( + topic.name, topic.metacode, topic.permission, + topic.desc, topic.link, x, y, topic.id + ); + }); + }, + + importSynapses: function(parsedSynapses) { + var self = Metamaps.Import; + + parsedSynapses.forEach(function(synapse) { + self.createSynapseWithParameters( + synapse.desc, synapse.category, synapse.permission, + synapse.topic1, synapse.topic2 + ); + }); + }, + + createTopicWithParameters: function(name, metacode_name, permission, desc, + link, xloc, yloc, import_id) { + $(document).trigger(Metamaps.Map.events.editedByActiveMapper); var metacode = Metamaps.Metacodes.where({name: metacode_name})[0] || null; if (metacode === null) return console.error("metacode not found"); @@ -90,7 +214,8 @@ Metamaps.Import = { metacode_id: metacode.id, permission: permission || Metamaps.Active.Map.get('permission'), desc: desc, - link: link + link: link, + import_id: import_id }); Metamaps.Topics.add(topic); @@ -103,6 +228,32 @@ Metamaps.Import = { Metamaps.Mappings.add(mapping); // this function also includes the creation of the topic in the database - self.renderTopic(mapping, topic, true, true); + Metamaps.Topic.renderTopic(mapping, topic, true, true); + + Metamaps.Famous.viz.hideInstructions(); + }, + + createSynapseWithParameters: function(description, category, permission, + node1_id, node2_id) { + var topic1 = Metamaps.Topics.where({import_id: node1_id}); + var topic2 = Metamaps.Topics.where({import_id: node2_id}); + var node1 = topic1.get('node'); + var node2 = topic2.get('node'); + // TODO check if topic1 and topic2 were sucessfully found... + + var synapse = new Metamaps.Backbone.Synapse({ + desc: description, + category: category, + permission: permission, + node1_id: node1_id, + node2_id: node2_id, + }); + + var mapping = new Metamaps.Backbone.Mapping({ + mappable_type: "Synapse", + mappable_id: synapse.cid, + }); + + Metamaps.Synapse.renderSynapse(mapping, synapse, node1, node2, true); }, }; From 61262aaec2cb4d4979918a375bd9bbc43c56b825 Mon Sep 17 00:00:00 2001 From: Devin Howard Date: Fri, 5 Feb 2016 17:49:59 +0800 Subject: [PATCH 04/13] implement csv/xls export --- app/models/map.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/map.rb b/app/models/map.rb index 5cd30bbe..696982da 100644 --- a/app/models/map.rb +++ b/app/models/map.rb @@ -101,7 +101,7 @@ class Map < ActiveRecord::Base end end end - + def decode_base64(imgBase64) decoded_data = Base64.decode64(imgBase64) From 8f532708ce32626d1d5a309f7568efed5ade968f Mon Sep 17 00:00:00 2001 From: Devin Howard Date: Fri, 12 Feb 2016 15:57:34 +0800 Subject: [PATCH 05/13] update xls/csv format to better serialize topics and synapses --- app/models/map.rb | 23 +++++++++++++++++++++-- app/views/maps/show.xls.erb | 35 ++++++++++++++++++++++++++++++----- 2 files changed, 51 insertions(+), 7 deletions(-) diff --git a/app/models/map.rb b/app/models/map.rb index 696982da..e786eb91 100644 --- a/app/models/map.rb +++ b/app/models/map.rb @@ -86,12 +86,17 @@ class Map < ActiveRecord::Base def to_csv(options = {}) CSV.generate(options) do |csv| - csv << ["id", "name", "metacode", "desc", "link", "user.name", "permission", "synapses"] - self.topics.each do |topic| + csv << ["topics"] + csv << ["id", "name", "metacode", "x", "y", "desc", "link", "user.name", "permission"] + self.topicmappings.each do |mapping| + topic = mapping.mappable + next if topic.nil? csv << [ topic.id, topic.name, topic.metacode.name, + mapping.x, + mapping.y, topic.desc, topic.link, topic.user.name, @@ -99,6 +104,20 @@ class Map < ActiveRecord::Base topic.synapses_csv("text") ] end + csv << [] + csv << ["synapses"] + csv << ["id", "description", "category", "topic1", "topic2", "username", "permission"] + self.synapses.each do |synapse| + csv << [ + synapse.id, + synapse.desc, + synapse.category, + synapse.node1_id, + synapse.node2_id, + synapse.user.name, + synapse.permission + ] + end end end diff --git a/app/views/maps/show.xls.erb b/app/views/maps/show.xls.erb index d00dd36e..4b22257e 100644 --- a/app/views/maps/show.xls.erb +++ b/app/views/maps/show.xls.erb @@ -1,26 +1,51 @@ + + + - - <% @map.topics.each do |topic| %> + <% @map.topicmappings.each do |mapping| %> + <% topic = mapping.mappable %> + <% next if topic.nil? %> + + - <% topic.synapses_csv.each do |s_text| %> - - <% end %> + + <% end %> + + + + + + + + + + + + <% @map.synapses.each do |synapse| %> + + + + + + + + <% end %>
Topics
ID Name MetacodeXY Description Link Username PermissionSynapses
<%= topic.id %> <%= topic.name %> <%= topic.metacode.name %><%= mapping.xloc %><%= mapping.yloc %> <%= topic.desc %> <%= topic.link %> <%= topic.user.name %> <%= topic.permission %><%= s_text %>
Synapses
IDDescriptionCategoryTopic1Topic2UsernamePermission
<%= synapse.id %><%= synapse.desc %><%= synapse.category %><%= synapse.node1_id %><%= synapse.node2_id %><%= synapse.user.name %><%= synapse.permission %>
From ea677f8a6b36993674d94ff45b16b24b4688ea73 Mon Sep 17 00:00:00 2001 From: Devin Howard Date: Sat, 13 Feb 2016 12:53:07 +0800 Subject: [PATCH 06/13] DRY up csv/xls rendering, put it into model --- app/models/map.rb | 69 ++++++++++++++++++++----------------- app/views/maps/show.xls.erb | 54 ++++------------------------- 2 files changed, 44 insertions(+), 79 deletions(-) diff --git a/app/models/map.rb b/app/models/map.rb index e786eb91..9efc278b 100644 --- a/app/models/map.rb +++ b/app/models/map.rb @@ -84,39 +84,46 @@ class Map < ActiveRecord::Base json end + def to_spreadsheet + spreadsheet = [] + spreadsheet << ["Topics"] + spreadsheet << ["Id", "Name", "Metacode", "X", "Y", "Description", "Link", "User", "Permission"] + self.topicmappings.each do |mapping| + topic = mapping.mappable + next if topic.nil? + spreadsheet << [ + topic.id, + topic.name, + topic.metacode.name, + mapping.xloc, + mapping.yloc, + topic.desc, + topic.link, + topic.user.name, + topic.permission + ] + end + spreadsheet << [] + spreadsheet << ["Synapses"] + spreadsheet << ["Id", "Description", "Category", "Topic1", "Topic2", "User", "Permission"] + self.synapses.each do |synapse| + spreadsheet << [ + synapse.id, + synapse.desc, + synapse.category, + synapse.node1_id, + synapse.node2_id, + synapse.user.name, + synapse.permission + ] + end + spreadsheet + end + def to_csv(options = {}) CSV.generate(options) do |csv| - csv << ["topics"] - csv << ["id", "name", "metacode", "x", "y", "desc", "link", "user.name", "permission"] - self.topicmappings.each do |mapping| - topic = mapping.mappable - next if topic.nil? - csv << [ - topic.id, - topic.name, - topic.metacode.name, - mapping.x, - mapping.y, - topic.desc, - topic.link, - topic.user.name, - topic.permission, - topic.synapses_csv("text") - ] - end - csv << [] - csv << ["synapses"] - csv << ["id", "description", "category", "topic1", "topic2", "username", "permission"] - self.synapses.each do |synapse| - csv << [ - synapse.id, - synapse.desc, - synapse.category, - synapse.node1_id, - synapse.node2_id, - synapse.user.name, - synapse.permission - ] + to_spreadsheet.each do |line| + csv << line end end end diff --git a/app/views/maps/show.xls.erb b/app/views/maps/show.xls.erb index 4b22257e..2f11a946 100644 --- a/app/views/maps/show.xls.erb +++ b/app/views/maps/show.xls.erb @@ -1,51 +1,9 @@ - - - - - - - - - - - - - <% @map.topicmappings.each do |mapping| %> - <% topic = mapping.mappable %> - <% next if topic.nil? %> - - - - - - - - - - - - <% end %> - - - - - - - - - - - - <% @map.synapses.each do |synapse| %> - - - - - - - - - + <% @map.to_spreadsheet.each do |line| %> + + <% line.each do |field| %> + + <% end %> + <% end %>
Topics
IDNameMetacodeXYDescriptionLinkUsernamePermission
<%= topic.id %><%= topic.name %><%= topic.metacode.name %><%= mapping.xloc %><%= mapping.yloc %><%= topic.desc %><%= topic.link %><%= topic.user.name %><%= topic.permission %>
Synapses
IDDescriptionCategoryTopic1Topic2UsernamePermission
<%= synapse.id %><%= synapse.desc %><%= synapse.category %><%= synapse.node1_id %><%= synapse.node2_id %><%= synapse.user.name %><%= synapse.permission %>
<%= field %>
From c77cc32734f31cbfdede99a11859811cdff3773a Mon Sep 17 00:00:00 2001 From: Devin Howard Date: Sun, 21 Feb 2016 16:28:31 +0800 Subject: [PATCH 07/13] import fixes - better abort logic & messaging - handle \r line delim - better example format at top --- .../javascripts/src/Metamaps.Import.js.erb | 64 ++++++++++++++----- 1 file changed, 47 insertions(+), 17 deletions(-) diff --git a/app/assets/javascripts/src/Metamaps.Import.js.erb b/app/assets/javascripts/src/Metamaps.Import.js.erb index b604e970..8a366560 100644 --- a/app/assets/javascripts/src/Metamaps.Import.js.erb +++ b/app/assets/javascripts/src/Metamaps.Import.js.erb @@ -2,19 +2,30 @@ * Example tab-separated input: * Some fields will be ignored * - * id name metacode desc link user.name permission synapses - * 1 topic1 Catalyst admin commons 1->7 - * 2 topic2 Event admin commons - * 5 topic Action admin commons - * 6 topic6 Action admin commons 6->7 - * 7 topic7 Action admin commons 7->8 7<-6 7<-1 - * 8 topic8 Action admin commons 8<-7 + * Topics + * Id Name Metacode X Y Description Link User Permission + * 8 topic8 Action -231 131 admin commons + * 5 topic Action -229 -131 admin commons + * 7 topic7.1 Action -470 -55 hey admin commons + * 2 topic2 Event -57 -63 admin commons + * 1 topic1 Catalyst -51 50 admin commons + * 6 topic6 Action -425 63 admin commons + * + * Synapses + * Id Description Category Topic1 Topic2 User Permission + * 43 from-to 6 2 admin commons + * 44 from-to 6 1 admin commons + * 45 from-to 6 5 admin commons + * 46 from-to 2 7 admin commons + * 47 from-to 8 6 admin commons + * 48 from-to 8 1 admin commons + * */ Metamaps.Import = { topicWhitelist: [ 'name', 'metacode', 'description', 'link', 'permission' - ], + ], synapseWhitelist: [ 'desc', 'description', 'category', 'topic1', 'topic2', 'permission' ], @@ -25,6 +36,8 @@ Metamaps.Import = { var text = e.originalEvent.clipboardData.getData('text/plain'); var results = self.parseTabbedString(text); + if (results === false) return; + var topics = results.topics; var synapses = results.synapses; @@ -39,9 +52,16 @@ Metamaps.Import = { }, abort: function(message) { + alert("Sorry, something went wrong!\n\n" + message); console.error(message); }, + simplify: function(string) { + return string + .replace(/(^\s*|\s*$)/g, '') + .toLowerCase(); + }, + parseTabbedString: function(text) { var self = Metamaps.Import; @@ -49,9 +69,12 @@ Metamaps.Import = { var delim = "\n"; if (text.indexOf("\r\n") !== -1) { delim = "\r\n"; + } else if (text.indexOf("\r") !== -1) { + delim = "\r"; }//if var STATES = { + ABORT: -1, UNKNOWN: 0, TOPICS_NEED_HEADERS: 1, SYNAPSES_NEED_HEADERS: 2, @@ -67,7 +90,7 @@ Metamaps.Import = { var synapseHeaders = []; lines.forEach(function(line_raw, index) { - var line = line_raw.split(' '); // tab character + var line = line_raw.split("\t"); var noblanks = line.filter(function(elt) { return elt !== ""; }); @@ -76,10 +99,10 @@ Metamaps.Import = { if (noblanks.length === 0) { state = STATES.UNKNOWN; break; - } else if (noblanks.length === 1 && line[0].toLowerCase() === 'topics') { + } else if (noblanks.length === 1 && self.simplify(line[0]) === 'topics') { state = STATES.TOPICS_NEED_HEADERS; break; - } else if (noblanks.length === 1 && line[0].toLowerCase() === 'synapses') { + } else if (noblanks.length === 1 && self.simplify(line[0]) === 'synapses') { state = STATES.SYNAPSES_NEED_HEADERS; break; } @@ -89,7 +112,8 @@ Metamaps.Import = { case STATES.TOPICS_NEED_HEADERS: if (noblanks.length < 2) { - return self.abort("Not enough topic headers on line " + index); + self.abort("Not enough topic headers on line " + index); + state = STATES.ABORT; } topicHeaders = line.map(function(header, index) { return header.toLowerCase().replace('description', 'desc'); @@ -99,7 +123,8 @@ Metamaps.Import = { case STATES.SYNAPSES_NEED_HEADERS: if (noblanks.length < 2) { - return self.abort("Not enough synapse headers on line " + index); + self.abort("Not enough synapse headers on line " + index); + state = STATES.ABORT; } synapseHeaders = line.map(function(header, index) { return header.toLowerCase().replace('description', 'desc'); @@ -148,14 +173,19 @@ Metamaps.Import = { results.synapses.push(synapse); } break; - + case STATES.ABORT: + ; default: - return self.abort("Invalid state while parsing import data. " + - "Check code."); + self.abort("Invalid state while parsing import data. Check code."); + state = STATES.ABORT; } }); - return results; + if (state === STATES.ABORT) { + return false; + } else { + return results; + } }, From 387c863222df6d877bee7801ee1f40750f259a9f Mon Sep 17 00:00:00 2001 From: Devin Howard Date: Sun, 21 Feb 2016 17:02:37 +0800 Subject: [PATCH 08/13] fix a bug with synapses and use cid to link new topics with synapses Synapses are now created client-side, but still rejected server-side --- .../javascripts/src/Metamaps.Import.js.erb | 26 +++++++++++-------- app/assets/javascripts/src/Metamaps.js.erb | 6 ++--- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/app/assets/javascripts/src/Metamaps.Import.js.erb b/app/assets/javascripts/src/Metamaps.Import.js.erb index 8a366560..8d6b3867 100644 --- a/app/assets/javascripts/src/Metamaps.Import.js.erb +++ b/app/assets/javascripts/src/Metamaps.Import.js.erb @@ -24,11 +24,12 @@ Metamaps.Import = { topicWhitelist: [ - 'name', 'metacode', 'description', 'link', 'permission' + 'id', 'name', 'metacode', 'description', 'link', 'permission' ], synapseWhitelist: [ - 'desc', 'description', 'category', 'topic1', 'topic2', 'permission' + 'id', 'desc', 'description', 'category', 'topic1', 'topic2', 'permission' ], + cidMappings: {}, //to be filled by import_id => cid mappings init: function() { var self = Metamaps.Import; @@ -145,7 +146,7 @@ Metamaps.Import = { var header = topicHeaders[index]; if (self.topicWhitelist.indexOf(header) === -1) return; topic[header] = field; - if (header === 'x' || header === 'y') { + if (['id', 'x', 'y'].indexOf(header) !== -1) { topic[header] = parseInt(topic[header]); }//if }); @@ -166,8 +167,8 @@ Metamaps.Import = { var header = synapseHeaders[index]; if (self.synapseWhitelist.indexOf(header) === -1) return; synapse[header] = field; - if (header === 'topic1' || header === 'topic2') { - synapse[header] = parseInt(header); + if (['id', 'topic1', 'topic2'].indexOf(header) !== -1) { + synapse[header] = parseInt(synapse[header]); }//if }); results.synapses.push(synapse); @@ -235,6 +236,7 @@ Metamaps.Import = { createTopicWithParameters: function(name, metacode_name, permission, desc, link, xloc, yloc, import_id) { + var self = Metamaps.Import; $(document).trigger(Metamaps.Map.events.editedByActiveMapper); var metacode = Metamaps.Metacodes.where({name: metacode_name})[0] || null; if (metacode === null) return console.error("metacode not found"); @@ -245,9 +247,9 @@ Metamaps.Import = { permission: permission || Metamaps.Active.Map.get('permission'), desc: desc, link: link, - import_id: import_id }); Metamaps.Topics.add(topic); + self.cidMappings[import_id] = topic.cid; var mapping = new Metamaps.Backbone.Mapping({ xloc: xloc, @@ -260,13 +262,15 @@ Metamaps.Import = { // this function also includes the creation of the topic in the database Metamaps.Topic.renderTopic(mapping, topic, true, true); + Metamaps.Famous.viz.hideInstructions(); }, createSynapseWithParameters: function(description, category, permission, node1_id, node2_id) { - var topic1 = Metamaps.Topics.where({import_id: node1_id}); - var topic2 = Metamaps.Topics.where({import_id: node2_id}); + var self = Metamaps.Import; + var topic1 = Metamaps.Topics.get(self.cidMappings[node1_id]); + var topic2 = Metamaps.Topics.get(self.cidMappings[node2_id]); var node1 = topic1.get('node'); var node2 = topic2.get('node'); // TODO check if topic1 and topic2 were sucessfully found... @@ -275,13 +279,13 @@ Metamaps.Import = { desc: description, category: category, permission: permission, - node1_id: node1_id, - node2_id: node2_id, + node1_id: node1.id, + node2_id: node2.id, }); var mapping = new Metamaps.Backbone.Mapping({ mappable_type: "Synapse", - mappable_id: synapse.cid, + mappable_id: synapse.id, }); Metamaps.Synapse.renderSynapse(mapping, synapse, node1, node2, true); diff --git a/app/assets/javascripts/src/Metamaps.js.erb b/app/assets/javascripts/src/Metamaps.js.erb index 2eb08b70..85b12d48 100644 --- a/app/assets/javascripts/src/Metamaps.js.erb +++ b/app/assets/javascripts/src/Metamaps.js.erb @@ -377,7 +377,7 @@ Metamaps.Backbone.init = function () { mappable_id: this.isNew() ? this.cid : this.id }); }, - createEdge: function () { + createEdge: function (providedMapping) { var mapping, mappingID; var synapseID = this.isNew() ? this.cid : this.id; @@ -391,7 +391,7 @@ Metamaps.Backbone.init = function () { }; if (Metamaps.Active.Map) { - mapping = this.getMapping(); + mapping = providedMapping || this.getMapping(); mappingID = mapping.isNew() ? mapping.cid : mapping.id; edge.data.$mappings = []; edge.data.$mappingIDs = [mappingID]; @@ -4614,7 +4614,7 @@ Metamaps.Synapse = { var edgeOnViz; - var newedge = synapse.createEdge(); + var newedge = synapse.createEdge(mapping); Metamaps.Visualize.mGraph.graph.addAdjacence(node1, node2, newedge.data); edgeOnViz = Metamaps.Visualize.mGraph.graph.getAdjacence(node1.id, node2.id); From 14bdc8546bab98851b938ef224d20d0fd5e04770 Mon Sep 17 00:00:00 2001 From: Devin Howard Date: Sat, 26 Mar 2016 10:12:32 +0800 Subject: [PATCH 09/13] metacodes#show routes --- app/controllers/metacodes_controller.rb | 14 +++++++++++++- config/routes.rb | 5 ++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/app/controllers/metacodes_controller.rb b/app/controllers/metacodes_controller.rb index 77f9ba54..3480c4cd 100644 --- a/app/controllers/metacodes_controller.rb +++ b/app/controllers/metacodes_controller.rb @@ -1,5 +1,5 @@ class MetacodesController < ApplicationController - before_action :require_admin, except: [:index] + before_action :require_admin, except: [:index, :show] # GET /metacodes # GET /metacodes.json @@ -18,6 +18,18 @@ class MetacodesController < ApplicationController end end + # GET /metacodes/1.json + # GET /metacodes/Action.json + # GET /metacodes/action.json + def show + @metacode = Metacode.where('DOWNCASE(name) = ?', downcase(params[:name])).first if params[:name] + @metacode = Metacode.find(params[:id]) unless @metacode + + respond_to do |format| + format.json { render json: @metacode } + end + end + # GET /metacodes/new # GET /metacodes/new.json def new diff --git a/config/routes.rb b/config/routes.rb index c68091ed..cbbc50f2 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -23,7 +23,10 @@ Metamaps::Application.routes.draw do resources :messages, only: [:show, :create, :update, :destroy] resources :mappings, except: [:index, :new, :edit] resources :metacode_sets, :except => [:show] - resources :metacodes, :except => [:show, :destroy] + + resources :metacodes, :except => [:destroy] + get 'metacodes/:name', to: 'metacodes#show' + resources :synapses, except: [:index, :new, :edit] resources :topics, except: [:index, :new, :edit] do get :autocomplete_topic, :on => :collection From d3649f1d26bf4d8e87ed2eb4399c0760e6b9dc7e Mon Sep 17 00:00:00 2001 From: Devin Howard Date: Sat, 26 Mar 2016 11:31:55 +0800 Subject: [PATCH 10/13] DRY map exporting with policy_scoping --- app/controllers/maps_controller.rb | 17 +++- app/models/map.rb | 44 ---------- app/services/map_export_service.rb | 84 +++++++++++++++++++ .../maps/{show.xls.erb => export.xls.erb} | 2 +- config/routes.rb | 2 + 5 files changed, 102 insertions(+), 47 deletions(-) create mode 100644 app/services/map_export_service.rb rename app/views/maps/{show.xls.erb => export.xls.erb} (74%) diff --git a/app/controllers/maps_controller.rb b/app/controllers/maps_controller.rb index fa743d93..762f2e99 100644 --- a/app/controllers/maps_controller.rb +++ b/app/controllers/maps_controller.rb @@ -85,11 +85,24 @@ class MapsController < ApplicationController respond_with(@allmappers, @allmappings, @allsynapses, @alltopics, @allmessages, @map) } format.json { render json: @map } - format.csv { send_data @map.to_csv } - format.xls + format.csv { redirect_to :export } + format.xls { redirect_to :export } end end + # GET maps/:id/export + def export + map = Map.find(params[:id]) + authorize map + exporter = MapExportService(current_user, map) + respond_to do |format| + format.json { render json: exporter.json } + format.csv { send_data exporter.csv } + format.xls { @spreadsheet = exporter.xls } + end + end + + # GET maps/:id/contains def contains @map = Map.find(params[:id]) diff --git a/app/models/map.rb b/app/models/map.rb index 9efc278b..d9eb6a18 100644 --- a/app/models/map.rb +++ b/app/models/map.rb @@ -84,50 +84,6 @@ class Map < ActiveRecord::Base json end - def to_spreadsheet - spreadsheet = [] - spreadsheet << ["Topics"] - spreadsheet << ["Id", "Name", "Metacode", "X", "Y", "Description", "Link", "User", "Permission"] - self.topicmappings.each do |mapping| - topic = mapping.mappable - next if topic.nil? - spreadsheet << [ - topic.id, - topic.name, - topic.metacode.name, - mapping.xloc, - mapping.yloc, - topic.desc, - topic.link, - topic.user.name, - topic.permission - ] - end - spreadsheet << [] - spreadsheet << ["Synapses"] - spreadsheet << ["Id", "Description", "Category", "Topic1", "Topic2", "User", "Permission"] - self.synapses.each do |synapse| - spreadsheet << [ - synapse.id, - synapse.desc, - synapse.category, - synapse.node1_id, - synapse.node2_id, - synapse.user.name, - synapse.permission - ] - end - spreadsheet - end - - def to_csv(options = {}) - CSV.generate(options) do |csv| - to_spreadsheet.each do |line| - csv << line - end - end - end - def decode_base64(imgBase64) decoded_data = Base64.decode64(imgBase64) diff --git a/app/services/map_export_service.rb b/app/services/map_export_service.rb new file mode 100644 index 00000000..30b6109a --- /dev/null +++ b/app/services/map_export_service.rb @@ -0,0 +1,84 @@ +class MapExportService < Struct.new(:user, :map) + def json + # marshal_dump turns OpenStruct into a Hash + { + topics: exportable_topics.map(:marshal_dump), + synapses: exportable_synapses.map(:marshal_dump) + } + end + + def csv(options = {}) + CSV.generate(options) do |csv| + to_spreadsheet.each do |line| + csv << line + end + end + end + + def xls + to_spreadsheet + end + + private + + def topic_headings + [:id, :name, :metacode, :x, :y, :description, :link, :user, :permission] + end + def synapse_headings + [:topic1, :topic2, :category, :description, :user, :permission] + end + + def exportable_topics + visible_topics ||= Pundit.policy_scope!(@user, @map.topics) + topic_mappings = Mapping.includes(mappable: [:metacode, :user]) + .where(mappable: visible_topics, map: @map) + topic_mappings.map do |mapping| + topic = mapping.mappable + OpenStruct.new( + id: topic.id, + name: topic.name, + metacode: topic.metacode.name, + x: mapping.xloc, + y: mapping.yloc, + description: topic.desc, + link: topic.link, + user: topic.user.name, + permission: topic.permission + ) + end + end + + def exportable_synapses + visible_synapses = Pundit.policy_scope!(@user, @map.synapses) + visible_synapses.map do |synapse| + OpenStruct.new( + topic1: synapse.node1_id, + topic2: synapse.node2_id, + category: synapse.category, + description: synapse.desc, + user: synapse.user.name, + permission: synapse.permission + ) + end + end + + def to_spreadsheet + spreadsheet = [] + spreadsheet << ["Topics"] + spreadsheet << topic_headings.map(:capitalize) + exportable_topics.each do |topics| + # convert exportable_topics into an array of arrays + topic_headings.map do { |h| topics.send(h) } + end + + spreadsheet << [] + spreadsheet << ["Synapses"] + spreadsheet << synapse_headings.map(:capitalize) + exportable_synapses.each do |synapse| + # convert exportable_synapses into an array of arrays + synapse_headings.map do { |h| synapse.send(h) } + end + + spreadsheet + end +end diff --git a/app/views/maps/show.xls.erb b/app/views/maps/export.xls.erb similarity index 74% rename from app/views/maps/show.xls.erb rename to app/views/maps/export.xls.erb index 2f11a946..7030d501 100644 --- a/app/views/maps/show.xls.erb +++ b/app/views/maps/export.xls.erb @@ -1,5 +1,5 @@ - <% @map.to_spreadsheet.each do |line| %> + <% @spreadsheet.each do |line| %> <% line.each do |field| %> diff --git a/config/routes.rb b/config/routes.rb index cbbc50f2..a9f82d9c 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -36,6 +36,8 @@ Metamaps::Application.routes.draw do get 'topics/:id/relatives', to: 'topics#relatives', as: :relatives resources :maps, except: [:index, :new, :edit] + get 'maps/:id/export', to: 'maps#export' + get 'explore/active', to: 'maps#activemaps' get 'explore/featured', to: 'maps#featuredmaps' get 'explore/mine', to: 'maps#mymaps' From 92f78aa56a729809eb60e1f05be13a9f33894e47 Mon Sep 17 00:00:00 2001 From: Devin Howard Date: Sat, 26 Mar 2016 12:49:26 +0800 Subject: [PATCH 11/13] update tsv code to handle new export code at the very least. next step will be allowing json input too --- .../javascripts/src/Metamaps.Import.js.erb | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/app/assets/javascripts/src/Metamaps.Import.js.erb b/app/assets/javascripts/src/Metamaps.Import.js.erb index 8d6b3867..92c9566d 100644 --- a/app/assets/javascripts/src/Metamaps.Import.js.erb +++ b/app/assets/javascripts/src/Metamaps.Import.js.erb @@ -12,22 +12,23 @@ * 6 topic6 Action -425 63 admin commons * * Synapses - * Id Description Category Topic1 Topic2 User Permission - * 43 from-to 6 2 admin commons - * 44 from-to 6 1 admin commons - * 45 from-to 6 5 admin commons - * 46 from-to 2 7 admin commons - * 47 from-to 8 6 admin commons - * 48 from-to 8 1 admin commons + * Topic1 Topic2 Category Description User Permission + * 6 2 from-to admin commons + * 6 1 from-to admin commons + * 6 5 from-to admin commons + * 2 7 from-to admin commons + * 8 6 from-to admin commons + * 8 1 from-to admin commons * */ Metamaps.Import = { + // note that user is not imported topicWhitelist: [ - 'id', 'name', 'metacode', 'description', 'link', 'permission' + 'id', 'name', 'metacode', 'x', 'y', 'description', 'link', 'permission' ], synapseWhitelist: [ - 'id', 'desc', 'description', 'category', 'topic1', 'topic2', 'permission' + 'topic1', 'topic2', 'category', 'desc', 'description', 'permission' ], cidMappings: {}, //to be filled by import_id => cid mappings From 53867caae83e5fe0b11c11489ed56cf8f96ad4fb Mon Sep 17 00:00:00 2001 From: Devin Howard Date: Sat, 26 Mar 2016 14:20:49 +0800 Subject: [PATCH 12/13] allow JSON or TSV parsing --- app/assets/javascripts/src/Metamaps.Import.js.erb | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/app/assets/javascripts/src/Metamaps.Import.js.erb b/app/assets/javascripts/src/Metamaps.Import.js.erb index 92c9566d..bfce14bd 100644 --- a/app/assets/javascripts/src/Metamaps.Import.js.erb +++ b/app/assets/javascripts/src/Metamaps.Import.js.erb @@ -37,7 +37,16 @@ Metamaps.Import = { $('body').bind('paste', function(e) { var text = e.originalEvent.clipboardData.getData('text/plain'); - var results = self.parseTabbedString(text); + var results; + if (text[0] === '{') { + try { + results = JSON.parse(text); + } catch (Error e) { + results = false; + } + } else { + results = self.parseTabbedString(text); + } if (results === false) return; var topics = results.topics; From ae9f4a51a24c44ecbbe8e205cf8310f1edb3b4d4 Mon Sep 17 00:00:00 2001 From: Devin Howard Date: Sat, 26 Mar 2016 15:21:55 +0800 Subject: [PATCH 13/13] fix a few embarassing errors - export is working --- app/controllers/maps_controller.rb | 6 +++--- app/policies/map_policy.rb | 4 ++++ app/services/map_export_service.rb | 18 +++++++++--------- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/app/controllers/maps_controller.rb b/app/controllers/maps_controller.rb index 762f2e99..131e5959 100644 --- a/app/controllers/maps_controller.rb +++ b/app/controllers/maps_controller.rb @@ -85,8 +85,8 @@ class MapsController < ApplicationController respond_with(@allmappers, @allmappings, @allsynapses, @alltopics, @allmessages, @map) } format.json { render json: @map } - format.csv { redirect_to :export } - format.xls { redirect_to :export } + format.csv { redirect_to action: :export, format: :csv } + format.xls { redirect_to action: :export, format: :xls } end end @@ -94,7 +94,7 @@ class MapsController < ApplicationController def export map = Map.find(params[:id]) authorize map - exporter = MapExportService(current_user, map) + exporter = MapExportService.new(current_user, map) respond_to do |format| format.json { render json: exporter.json } format.csv { send_data exporter.csv } diff --git a/app/policies/map_policy.rb b/app/policies/map_policy.rb index 65f721bf..5b4bbfa9 100644 --- a/app/policies/map_policy.rb +++ b/app/policies/map_policy.rb @@ -31,6 +31,10 @@ class MapPolicy < ApplicationPolicy record.permission == 'commons' || record.permission == 'public' || record.user == user end + def export? + show? + end + def contains? show? end diff --git a/app/services/map_export_service.rb b/app/services/map_export_service.rb index 30b6109a..94a0cf17 100644 --- a/app/services/map_export_service.rb +++ b/app/services/map_export_service.rb @@ -2,8 +2,8 @@ class MapExportService < Struct.new(:user, :map) def json # marshal_dump turns OpenStruct into a Hash { - topics: exportable_topics.map(:marshal_dump), - synapses: exportable_synapses.map(:marshal_dump) + topics: exportable_topics.map(&:marshal_dump), + synapses: exportable_synapses.map(&:marshal_dump) } end @@ -29,9 +29,9 @@ class MapExportService < Struct.new(:user, :map) end def exportable_topics - visible_topics ||= Pundit.policy_scope!(@user, @map.topics) + visible_topics ||= Pundit.policy_scope!(user, map.topics) topic_mappings = Mapping.includes(mappable: [:metacode, :user]) - .where(mappable: visible_topics, map: @map) + .where(mappable: visible_topics, map: map) topic_mappings.map do |mapping| topic = mapping.mappable OpenStruct.new( @@ -49,7 +49,7 @@ class MapExportService < Struct.new(:user, :map) end def exportable_synapses - visible_synapses = Pundit.policy_scope!(@user, @map.synapses) + visible_synapses = Pundit.policy_scope!(user, map.synapses) visible_synapses.map do |synapse| OpenStruct.new( topic1: synapse.node1_id, @@ -65,18 +65,18 @@ class MapExportService < Struct.new(:user, :map) def to_spreadsheet spreadsheet = [] spreadsheet << ["Topics"] - spreadsheet << topic_headings.map(:capitalize) + spreadsheet << topic_headings.map(&:capitalize) exportable_topics.each do |topics| # convert exportable_topics into an array of arrays - topic_headings.map do { |h| topics.send(h) } + spreadsheet << topic_headings.map { |h| topics.send(h) } end spreadsheet << [] spreadsheet << ["Synapses"] - spreadsheet << synapse_headings.map(:capitalize) + spreadsheet << synapse_headings.map(&:capitalize) exportable_synapses.each do |synapse| # convert exportable_synapses into an array of arrays - synapse_headings.map do { |h| synapse.send(h) } + spreadsheet << synapse_headings.map { |h| synapse.send(h) } end spreadsheet
<%= field %>