api: import from commento export format

JSON data can be imported to restore previously exported data
or to migrate data from another self-hosted commento instance.

Closes https://gitlab.com/commento/commento/issues/239

api/domain_import_commento.go

@@ -0,0 +1,168 @@
+package main
+
+import (
+	"bytes"
+	"compress/gzip"
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+)
+
+type dataImport struct {
+	Version    int         `json:"version"`
+	Comments   []comment   `json:"comments"`
+	Commenters []commenter `json:"commenters"`
+}
+
+func domainImportCommento(domain string, url string) (int, error) {
+	if domain == "" || url == "" {
+		return 0, errorMissingField
+	}
+
+	resp, err := http.Get(url)
+	if err != nil {
+		logger.Errorf("cannot get url: %v", err)
+		return 0, errorCannotDownloadCommento
+	}
+
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		logger.Errorf("cannot read body: %v", err)
+		return 0, errorCannotDownloadCommento
+	}
+
+	zr, err := gzip.NewReader(bytes.NewBuffer(body))
+	if err != nil {
+		logger.Errorf("cannot create gzip reader: %v", err)
+		return 0, errorInternal
+	}
+
+	contents, err := ioutil.ReadAll(zr)
+	if err != nil {
+		logger.Errorf("cannot read gzip contents uncompressed: %v", err)
+		return 0, errorInternal
+	}
+
+	var data dataImport
+	if err := json.Unmarshal(contents, &data); err != nil {
+		logger.Errorf("cannot unmarshal JSON at %s: %v", url, err)
+		return 0, errorInternal
+	}
+
+	if data.Version != 1 {
+		logger.Errorf("invalid data version (got %d, want 1): %v", data.Version, err)
+		return 0, errorUnsupportedCommentoImportVersion
+	}
+
+	// Check if imported commentedHex or email exists, creating a map of
+	// commenterHex (old hex, new hex)
+	commenterHex := map[string]string{"anonymous": "anonymous"}
+	for _, commenter := range data.Commenters {
+		c, err := commenterGetByEmail("commento", commenter.Email)
+		if err != nil && err != errorNoSuchCommenter {
+			logger.Errorf("cannot get commenter by email: %v", err)
+			return 0, errorInternal
+		}
+
+		if err == nil {
+			commenterHex[commenter.CommenterHex] = c.CommenterHex
+			continue
+		}
+
+		randomPassword, err := randomHex(32)
+		if err != nil {
+			logger.Errorf("cannot generate random password for new commenter: %v", err)
+			return 0, errorInternal
+		}
+
+		commenterHex[commenter.CommenterHex], err = commenterNew(commenter.Email,
+			commenter.Name, commenter.Link, commenter.Photo, "commento", randomPassword)
+		if err != nil {
+			return 0, err
+		}
+	}
+
+	// Create a map of (parent hex, comments)
+	comments := make(map[string][]comment)
+	for _, comment := range data.Comments {
+		parentHex := comment.ParentHex
+		comments[parentHex] = append(comments[parentHex], comment)
+	}
+
+	// Import comments, creating a map of comment hex (old hex, new hex)
+	commentHex := map[string]string{"root": "root"}
+	numImported := 0
+	keys := []string{"root"}
+	for i := 0; i < len(keys); i++ {
+		for _, comment := range comments[keys[i]] {
+			cHex, ok := commenterHex[comment.CommenterHex]
+			if !ok {
+				logger.Errorf("cannot get commenter: %v", err)
+				return numImported, errorInternal
+			}
+			parentHex, ok := commentHex[comment.ParentHex]
+			if !ok {
+				logger.Errorf("cannot get parent comment: %v", err)
+				return numImported, errorInternal
+			}
+
+			hex, err := commentNew(
+				cHex,
+				domain,
+				comment.Path,
+				parentHex,
+				comment.Markdown,
+				comment.State,
+				comment.CreationDate)
+			if err != nil {
+				return numImported, err
+			}
+			commentHex[comment.CommentHex] = hex
+			numImported++
+			keys = append(keys, comment.CommentHex)
+		}
+	}
+
+	return numImported, nil
+}
+
+func domainImportCommentoHandler(w http.ResponseWriter, r *http.Request) {
+	type request struct {
+		OwnerToken *string `json:"ownerToken"`
+		Domain     *string `json:"domain"`
+		URL        *string `json:"url"`
+	}
+
+	var x request
+	if err := bodyUnmarshal(r, &x); err != nil {
+		bodyMarshal(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	o, err := ownerGetByOwnerToken(*x.OwnerToken)
+	if err != nil {
+		bodyMarshal(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	domain := domainStrip(*x.Domain)
+	isOwner, err := domainOwnershipVerify(o.OwnerHex, domain)
+	if err != nil {
+		bodyMarshal(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	if !isOwner {
+		bodyMarshal(w, response{"success": false, "message": errorNotAuthorised.Error()})
+		return
+	}
+
+	numImported, err := domainImportCommento(domain, *x.URL)
+	if err != nil {
+		bodyMarshal(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	bodyMarshal(w, response{"success": true, "numImported": numImported})
+}

api/domain_import_commento_test.go

@@ -0,0 +1,121 @@
+package main
+
+import (
+	"compress/gzip"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"testing"
+	"time"
+)
+
+func TestImportCommento(t *testing.T) {
+	failTestOnError(t, setupTestEnv())
+
+	// Create JSON data
+	data := dataImport{
+		Version: 1,
+		Comments: []comment{
+			{
+				CommentHex:   "5a349182b3b8e25107ab2b12e514f40fe0b69160a334019491d7c204aff6fdc2",
+				Domain:       "localhost:1313",
+				Path:         "/post/first-post/",
+				CommenterHex: "anonymous",
+				Markdown:     "This is a reply!",
+				Html:         "",
+				ParentHex:    "7ed60b1227f6c4850258a2ac0304e1936770117d6f3a379655f775c46b9f13cd",
+				Score:        0,
+				State:        "approved",
+				CreationDate: timeParse(t, "2020-01-27T14:08:44.061525Z"),
+				Direction:    0,
+				Deleted:      false,
+			},
+			{
+				CommentHex:   "7ed60b1227f6c4850258a2ac0304e1936770117d6f3a379655f775c46b9f13cd",
+				Domain:       "localhost:1313",
+				Path:         "/post/first-post/",
+				CommenterHex: "anonymous",
+				Markdown:     "This is a comment!",
+				Html:         "",
+				ParentHex:    "root",
+				Score:        0,
+				State:        "approved",
+				CreationDate: timeParse(t, "2020-01-27T14:07:49.244432Z"),
+				Direction:    0,
+				Deleted:      false,
+			},
+			{
+				CommentHex:   "a7c84f251b5a09d5b65e902cbe90633646437acefa3a52b761fee94002ac54c7",
+				Domain:       "localhost:1313",
+				Path:         "/post/first-post/",
+				CommenterHex: "4629a8216538b73987597d66f266c1a1801b0451f99cf066e7122aa104ef3b07",
+				Markdown:     "This is a test comment, bar foo\n\n#Here is something big\n\n```\nhere code();\n```",
+				Html:         "",
+				ParentHex:    "root",
+				Score:        0,
+				State:        "approved",
+				CreationDate: timeParse(t, "2020-01-27T14:20:21.101653Z"),
+				Direction:    0,
+				Deleted:      false,
+			},
+		},
+		Commenters: []commenter{
+			{
+				CommenterHex: "4629a8216538b73987597d66f266c1a1801b0451f99cf066e7122aa104ef3b07",
+				Email:        "john@doe.com",
+				Name:         "John Doe",
+				Link:         "https://john.doe",
+				Photo:        "undefined",
+				Provider:     "commento",
+				JoinDate:     timeParse(t, "2020-01-27T14:17:59.298737Z"),
+				IsModerator:  false,
+			},
+		},
+	}
+
+	// Create listener with random port
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Errorf("couldn't create listener: %v", err)
+		return
+	}
+	defer func() {
+		_ = listener.Close()
+	}()
+	port := listener.Addr().(*net.TCPAddr).Port
+
+	// Launch http server serving commento json gzipped data
+	go func() {
+		http.Serve(listener, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			gzipper := gzip.NewWriter(w)
+			defer func() {
+				_ = gzipper.Close()
+			}()
+			encoder := json.NewEncoder(gzipper)
+			if err := encoder.Encode(data); err != nil {
+				t.Errorf("couldn't write data: %v", err)
+			}
+		}))
+	}()
+	url := fmt.Sprintf("http://127.0.0.1:%d", port)
+
+	domainNew("temp-owner-hex", "Example", "example.com")
+
+	n, err := domainImportCommento("example.com", url)
+	if err != nil {
+		t.Errorf("unexpected error importing comments: %v", err)
+		return
+	}
+	if n != len(data.Comments) {
+		t.Errorf("imported comments missmatch (got %d, want %d)", n, len(data.Comments))
+	}
+}
+
+func timeParse(t *testing.T, s string) time.Time {
+	time, err := time.Parse(time.RFC3339Nano, s)
+	if err != nil {
+		t.Errorf("couldn't parse time: %v", err)
+	}
+	return time
+}

api/errors.go

@@ -37,6 +37,7 @@ var errorNotModerator = errors.New("You need to be a moderator to do that.")
 var errorNotADirectory = errors.New("The given path is not a directory.")
 var errorGzip = errors.New("Cannot GZip content.")
 var errorCannotDownloadDisqus = errors.New("We could not download your Disqus export file.")
+var errorCannotDownloadCommento = errors.New("We could not download your Commento export file.")
 var errorSelfVote = errors.New("You cannot vote on your own comment.")
 var errorInvalidConfigFile = errors.New("Invalid config file.")
 var errorInvalidConfigValue = errors.New("Invalid config value.")
@@ -50,3 +51,4 @@ var errorInvalidEntity = errors.New("That entity does not exist.")
 var errorCannotDeleteOwnerWithActiveDomains = errors.New("You cannot delete your account until all domains associated with your account are deleted.")
 var errorNoSuchOwner = errors.New("No such owner.")
 var errorCannotUpdateOauthProfile = errors.New("You cannot update the profile of an external account managed by third-party log in. Please use the appropriate platform to update your details.")
+var errorUnsupportedCommentoImportVersion = errors.New("Unsupported Commento import format version.")

api/router_api.go

@@ -21,6 +21,7 @@ func apiRouterInit(router *mux.Router) error {
 	router.HandleFunc("/api/domain/moderator/delete", domainModeratorDeleteHandler).Methods("POST")
 	router.HandleFunc("/api/domain/statistics", domainStatisticsHandler).Methods("POST")
 	router.HandleFunc("/api/domain/import/disqus", domainImportDisqusHandler).Methods("POST")
+	router.HandleFunc("/api/domain/import/commento", domainImportCommentoHandler).Methods("POST")
 	router.HandleFunc("/api/domain/export/begin", domainExportBeginHandler).Methods("POST")
 	router.HandleFunc("/api/domain/export/download", domainExportDownloadHandler).Methods("GET")
 

api/testing.go

@@ -2,9 +2,10 @@ package main
 
 import (
 	"fmt"
-	"github.com/op/go-logging"
 	"os"
 	"testing"
+
+	"github.com/op/go-logging"
 )
 
 func failTestOnError(t *testing.T, err error) {

frontend/dashboard.html

@@ -390,10 +390,11 @@
             <div class="tabs-container">
               <div class="tab">
                 <ul class="tabs">
-                  <li class="tab-link original current" data-tab="install-tab-1">Disqus</li>
+                  <li class="tab-link original current" data-tab="import-tab-1">Disqus</li>
+                  <li class="tab-link" data-tab="import-tab-2">Commento</li>
                 </ul>
 
-                <div id="install-tab-1" class="content original current">
+                <div id="import-tab-1" class="content original current">
                   <div class="normal-text">
                     If you're currently using Disqus, you can import all comments into Commento:
                     <ul>
@@ -433,6 +434,41 @@
                     </ul>
                   </div>
                 </div>
+
+                <div id="import-tab-2" class="content">
+                  <div class="normal-text">
+                    If you've previously exported data from Commento you can restore it:
+                    <ul>
+                      <li>
+                        Upload your data file somewhere in the cloud and generate a shared link for it.
+                      </li>
+
+                      <li>
+                        Copy and paste that link here to start the import process:
+
+                        <br><br>
+
+                        <div class="commento-email-container">
+                          <div class="commento-email">
+                            <input class="commento-input" type="text" id="commento-url" placeholder="https://drive.google.com/uc?export=download&id=...">
+                            <button id="commento-import-button" class="commento-email-button" onclick="window.commento.importCommento()">Import</button>
+                          </div>
+                        </div>
+
+                        <br>
+                      </li>
+
+                      <li>
+                        Commento will automatically download this file, extract it, parse it and import comments into Commento. URL information, comment authors, text formatting, and nested replies will be preserved.
+                      </li>
+                        
+                      <li>
+                        It is strongly recommended you do this only once. Importing multiple times may have unintended effects.
+                      </li>
+                    </ul>
+                  </div>
+                </div>
+
               </div>
             </div>
           </div>

frontend/js/dashboard-import.js

@@ -34,4 +34,29 @@
     });
   }
 
+  global.importCommento = function() {
+    var url = $("#commento-url").val();
+    var data = global.dashboard.$data;
+
+    var json = {
+      "ownerToken": global.cookieGet("commentoOwnerToken"),
+      "domain": data.domains[data.cd].domain,
+      "url": url,
+    }
+
+    global.buttonDisable("#commento-import-button");
+    global.post(global.origin + "/api/domain/import/commento", json, function(resp) {
+      global.buttonEnable("#commento-import-button");
+
+      if (!resp.success) {
+        global.globalErrorShow(resp.message);
+        return;
+      }
+
+      $("#commento-import-button").hide();
+
+      global.globalOKShow("Imported " + resp.numImported + " comments!");
+    });
+  }
+
 } (window.commento, document));