From 0d929595cc7001c7d7cc76f5d266cf4eb67da477 Mon Sep 17 00:00:00 2001 From: igolaizola <5315129-igolaizola@users.noreply.gitlab.com> Date: Sat, 1 Feb 2020 08:12:21 +0100 Subject: [PATCH] api: import from commento export format JSON data can be imported to restore previously exported data or to migrate data from another self-hosted commento instance. Closes https://gitlab.com/commento/commento/issues/239 --- api/domain_import_commento.go | 168 +++++++++++++++++++++++++++++ api/domain_import_commento_test.go | 121 +++++++++++++++++++++ api/errors.go | 2 + api/router_api.go | 1 + api/testing.go | 3 +- frontend/dashboard.html | 40 ++++++- frontend/js/dashboard-import.js | 25 +++++ 7 files changed, 357 insertions(+), 3 deletions(-) create mode 100644 api/domain_import_commento.go create mode 100644 api/domain_import_commento_test.go diff --git a/api/domain_import_commento.go b/api/domain_import_commento.go new file mode 100644 index 0000000..a09da73 --- /dev/null +++ b/api/domain_import_commento.go @@ -0,0 +1,168 @@ +package main + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "io/ioutil" + "net/http" +) + +type dataImport struct { + Version int `json:"version"` + Comments []comment `json:"comments"` + Commenters []commenter `json:"commenters"` +} + +func domainImportCommento(domain string, url string) (int, error) { + if domain == "" || url == "" { + return 0, errorMissingField + } + + resp, err := http.Get(url) + if err != nil { + logger.Errorf("cannot get url: %v", err) + return 0, errorCannotDownloadCommento + } + + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + logger.Errorf("cannot read body: %v", err) + return 0, errorCannotDownloadCommento + } + + zr, err := gzip.NewReader(bytes.NewBuffer(body)) + if err != nil { + logger.Errorf("cannot create gzip reader: %v", err) + return 0, errorInternal + } + + contents, err := ioutil.ReadAll(zr) + if err != nil { + logger.Errorf("cannot read gzip contents uncompressed: %v", err) + return 0, errorInternal + } + + var data dataImport + if err := json.Unmarshal(contents, &data); err != nil { + logger.Errorf("cannot unmarshal JSON at %s: %v", url, err) + return 0, errorInternal + } + + if data.Version != 1 { + logger.Errorf("invalid data version (got %d, want 1): %v", data.Version, err) + return 0, errorUnsupportedCommentoImportVersion + } + + // Check if imported commentedHex or email exists, creating a map of + // commenterHex (old hex, new hex) + commenterHex := map[string]string{"anonymous": "anonymous"} + for _, commenter := range data.Commenters { + c, err := commenterGetByEmail("commento", commenter.Email) + if err != nil && err != errorNoSuchCommenter { + logger.Errorf("cannot get commenter by email: %v", err) + return 0, errorInternal + } + + if err == nil { + commenterHex[commenter.CommenterHex] = c.CommenterHex + continue + } + + randomPassword, err := randomHex(32) + if err != nil { + logger.Errorf("cannot generate random password for new commenter: %v", err) + return 0, errorInternal + } + + commenterHex[commenter.CommenterHex], err = commenterNew(commenter.Email, + commenter.Name, commenter.Link, commenter.Photo, "commento", randomPassword) + if err != nil { + return 0, err + } + } + + // Create a map of (parent hex, comments) + comments := make(map[string][]comment) + for _, comment := range data.Comments { + parentHex := comment.ParentHex + comments[parentHex] = append(comments[parentHex], comment) + } + + // Import comments, creating a map of comment hex (old hex, new hex) + commentHex := map[string]string{"root": "root"} + numImported := 0 + keys := []string{"root"} + for i := 0; i < len(keys); i++ { + for _, comment := range comments[keys[i]] { + cHex, ok := commenterHex[comment.CommenterHex] + if !ok { + logger.Errorf("cannot get commenter: %v", err) + return numImported, errorInternal + } + parentHex, ok := commentHex[comment.ParentHex] + if !ok { + logger.Errorf("cannot get parent comment: %v", err) + return numImported, errorInternal + } + + hex, err := commentNew( + cHex, + domain, + comment.Path, + parentHex, + comment.Markdown, + comment.State, + comment.CreationDate) + if err != nil { + return numImported, err + } + commentHex[comment.CommentHex] = hex + numImported++ + keys = append(keys, comment.CommentHex) + } + } + + return numImported, nil +} + +func domainImportCommentoHandler(w http.ResponseWriter, r *http.Request) { + type request struct { + OwnerToken *string `json:"ownerToken"` + Domain *string `json:"domain"` + URL *string `json:"url"` + } + + var x request + if err := bodyUnmarshal(r, &x); err != nil { + bodyMarshal(w, response{"success": false, "message": err.Error()}) + return + } + + o, err := ownerGetByOwnerToken(*x.OwnerToken) + if err != nil { + bodyMarshal(w, response{"success": false, "message": err.Error()}) + return + } + + domain := domainStrip(*x.Domain) + isOwner, err := domainOwnershipVerify(o.OwnerHex, domain) + if err != nil { + bodyMarshal(w, response{"success": false, "message": err.Error()}) + return + } + + if !isOwner { + bodyMarshal(w, response{"success": false, "message": errorNotAuthorised.Error()}) + return + } + + numImported, err := domainImportCommento(domain, *x.URL) + if err != nil { + bodyMarshal(w, response{"success": false, "message": err.Error()}) + return + } + + bodyMarshal(w, response{"success": true, "numImported": numImported}) +} diff --git a/api/domain_import_commento_test.go b/api/domain_import_commento_test.go new file mode 100644 index 0000000..9f31690 --- /dev/null +++ b/api/domain_import_commento_test.go @@ -0,0 +1,121 @@ +package main + +import ( + "compress/gzip" + "encoding/json" + "fmt" + "net" + "net/http" + "testing" + "time" +) + +func TestImportCommento(t *testing.T) { + failTestOnError(t, setupTestEnv()) + + // Create JSON data + data := dataImport{ + Version: 1, + Comments: []comment{ + { + CommentHex: "5a349182b3b8e25107ab2b12e514f40fe0b69160a334019491d7c204aff6fdc2", + Domain: "localhost:1313", + Path: "/post/first-post/", + CommenterHex: "anonymous", + Markdown: "This is a reply!", + Html: "", + ParentHex: "7ed60b1227f6c4850258a2ac0304e1936770117d6f3a379655f775c46b9f13cd", + Score: 0, + State: "approved", + CreationDate: timeParse(t, "2020-01-27T14:08:44.061525Z"), + Direction: 0, + Deleted: false, + }, + { + CommentHex: "7ed60b1227f6c4850258a2ac0304e1936770117d6f3a379655f775c46b9f13cd", + Domain: "localhost:1313", + Path: "/post/first-post/", + CommenterHex: "anonymous", + Markdown: "This is a comment!", + Html: "", + ParentHex: "root", + Score: 0, + State: "approved", + CreationDate: timeParse(t, "2020-01-27T14:07:49.244432Z"), + Direction: 0, + Deleted: false, + }, + { + CommentHex: "a7c84f251b5a09d5b65e902cbe90633646437acefa3a52b761fee94002ac54c7", + Domain: "localhost:1313", + Path: "/post/first-post/", + CommenterHex: "4629a8216538b73987597d66f266c1a1801b0451f99cf066e7122aa104ef3b07", + Markdown: "This is a test comment, bar foo\n\n#Here is something big\n\n```\nhere code();\n```", + Html: "", + ParentHex: "root", + Score: 0, + State: "approved", + CreationDate: timeParse(t, "2020-01-27T14:20:21.101653Z"), + Direction: 0, + Deleted: false, + }, + }, + Commenters: []commenter{ + { + CommenterHex: "4629a8216538b73987597d66f266c1a1801b0451f99cf066e7122aa104ef3b07", + Email: "john@doe.com", + Name: "John Doe", + Link: "https://john.doe", + Photo: "undefined", + Provider: "commento", + JoinDate: timeParse(t, "2020-01-27T14:17:59.298737Z"), + IsModerator: false, + }, + }, + } + + // Create listener with random port + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Errorf("couldn't create listener: %v", err) + return + } + defer func() { + _ = listener.Close() + }() + port := listener.Addr().(*net.TCPAddr).Port + + // Launch http server serving commento json gzipped data + go func() { + http.Serve(listener, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gzipper := gzip.NewWriter(w) + defer func() { + _ = gzipper.Close() + }() + encoder := json.NewEncoder(gzipper) + if err := encoder.Encode(data); err != nil { + t.Errorf("couldn't write data: %v", err) + } + })) + }() + url := fmt.Sprintf("http://127.0.0.1:%d", port) + + domainNew("temp-owner-hex", "Example", "example.com") + + n, err := domainImportCommento("example.com", url) + if err != nil { + t.Errorf("unexpected error importing comments: %v", err) + return + } + if n != len(data.Comments) { + t.Errorf("imported comments missmatch (got %d, want %d)", n, len(data.Comments)) + } +} + +func timeParse(t *testing.T, s string) time.Time { + time, err := time.Parse(time.RFC3339Nano, s) + if err != nil { + t.Errorf("couldn't parse time: %v", err) + } + return time +} diff --git a/api/errors.go b/api/errors.go index 3ccebc8..bc9710a 100644 --- a/api/errors.go +++ b/api/errors.go @@ -37,6 +37,7 @@ var errorNotModerator = errors.New("You need to be a moderator to do that.") var errorNotADirectory = errors.New("The given path is not a directory.") var errorGzip = errors.New("Cannot GZip content.") var errorCannotDownloadDisqus = errors.New("We could not download your Disqus export file.") +var errorCannotDownloadCommento = errors.New("We could not download your Commento export file.") var errorSelfVote = errors.New("You cannot vote on your own comment.") var errorInvalidConfigFile = errors.New("Invalid config file.") var errorInvalidConfigValue = errors.New("Invalid config value.") @@ -50,3 +51,4 @@ var errorInvalidEntity = errors.New("That entity does not exist.") var errorCannotDeleteOwnerWithActiveDomains = errors.New("You cannot delete your account until all domains associated with your account are deleted.") var errorNoSuchOwner = errors.New("No such owner.") var errorCannotUpdateOauthProfile = errors.New("You cannot update the profile of an external account managed by third-party log in. Please use the appropriate platform to update your details.") +var errorUnsupportedCommentoImportVersion = errors.New("Unsupported Commento import format version.") diff --git a/api/router_api.go b/api/router_api.go index 98ecf28..3511e82 100644 --- a/api/router_api.go +++ b/api/router_api.go @@ -21,6 +21,7 @@ func apiRouterInit(router *mux.Router) error { router.HandleFunc("/api/domain/moderator/delete", domainModeratorDeleteHandler).Methods("POST") router.HandleFunc("/api/domain/statistics", domainStatisticsHandler).Methods("POST") router.HandleFunc("/api/domain/import/disqus", domainImportDisqusHandler).Methods("POST") + router.HandleFunc("/api/domain/import/commento", domainImportCommentoHandler).Methods("POST") router.HandleFunc("/api/domain/export/begin", domainExportBeginHandler).Methods("POST") router.HandleFunc("/api/domain/export/download", domainExportDownloadHandler).Methods("GET") diff --git a/api/testing.go b/api/testing.go index 96c0ecf..6112b3c 100644 --- a/api/testing.go +++ b/api/testing.go @@ -2,9 +2,10 @@ package main import ( "fmt" - "github.com/op/go-logging" "os" "testing" + + "github.com/op/go-logging" ) func failTestOnError(t *testing.T, err error) { diff --git a/frontend/dashboard.html b/frontend/dashboard.html index 2e9db6e..c1e9a07 100644 --- a/frontend/dashboard.html +++ b/frontend/dashboard.html @@ -390,10 +390,11 @@
-
+
If you're currently using Disqus, you can import all comments into Commento:
    @@ -433,6 +434,41 @@
+ +
+
+ If you've previously exported data from Commento you can restore it: +
    +
  • + Upload your data file somewhere in the cloud and generate a shared link for it. +
    • + +
  • + Copy and paste that link here to start the import process: + +

    + +
    +
    + + +
    +
    + +
    +
    • + +
  • + Commento will automatically download this file, extract it, parse it and import comments into Commento. URL information, comment authors, text formatting, and nested replies will be preserved. +
    • + +
  • + It is strongly recommended you do this only once. Importing multiple times may have unintended effects. +
    • +
+
+
+
diff --git a/frontend/js/dashboard-import.js b/frontend/js/dashboard-import.js index 7850ca7..c27b9f0 100644 --- a/frontend/js/dashboard-import.js +++ b/frontend/js/dashboard-import.js @@ -34,4 +34,29 @@ }); } + global.importCommento = function() { + var url = $("#commento-url").val(); + var data = global.dashboard.$data; + + var json = { + "ownerToken": global.cookieGet("commentoOwnerToken"), + "domain": data.domains[data.cd].domain, + "url": url, + } + + global.buttonDisable("#commento-import-button"); + global.post(global.origin + "/api/domain/import/commento", json, function(resp) { + global.buttonEnable("#commento-import-button"); + + if (!resp.success) { + global.globalErrorShow(resp.message); + return; + } + + $("#commento-import-button").hide(); + + global.globalOKShow("Imported " + resp.numImported + " comments!"); + }); + } + } (window.commento, document));