From 57e5bc7abcb90b2409cb9a395d181f66c87dd910 Mon Sep 17 00:00:00 2001 From: Adhityaa Date: Thu, 14 Jun 2018 14:28:11 +0530 Subject: [PATCH] api: add disqus endpoint --- api/domain_import_disqus.go | 208 ++++++++++++++++++++++++++++++++ api/errors.go | 1 + api/router_api.go | 1 + frontend/dashboard.html | 4 +- frontend/js/dashboard-import.js | 4 +- 5 files changed, 215 insertions(+), 3 deletions(-) create mode 100644 api/domain_import_disqus.go diff --git a/api/domain_import_disqus.go b/api/domain_import_disqus.go new file mode 100644 index 0000000..c73ae74 --- /dev/null +++ b/api/domain_import_disqus.go @@ -0,0 +1,208 @@ +package main + +import ( + "time" + "io/ioutil" + "net/http" + "compress/gzip" + "encoding/xml" + // "github.com/grokify/html-strip-tags-go" + "github.com/lunny/html2md" +) + +type disqusThread struct { + XMLName xml.Name `xml:"thread"` + Id string `xml:"http://disqus.com/disqus-internals id,attr"` + URL string `xml:"link"` + Name string `xml:"name"` +} + +type disqusAuthor struct { + XMLName xml.Name `xml:"author"` + IsAnonymous bool `xml:"isAnonymous"` + Name string `xml:"name"` + Email string `xml:"email"` +} + +type disqusThreadId struct { + XMLName xml.Name `xml:"thread"` + Id string `xml:"http://disqus.com/disqus-internals id,attr"` +} + +type disqusParentId struct { + XMLName xml.Name `xml:"parent"` + Id string `xml:"http://disqus.com/disqus-internals id,attr"` +} + +type disqusPostId struct { + XMLName xml.Name `xml:"post"` + Id string `xml:"http://disqus.com/disqus-internals id,attr"` +} + +type disqusPost struct { + XMLName xml.Name `xml:"post"` + Id string `xml:"http://disqus.com/disqus-internals id,attr"` + ThreadId disqusThreadId `xml:"thread"` + ParentId disqusParentId `xml:"parent"` + PostId disqusPostId `xml:"post"` + Message string `xml:"message"` + CreationDate time.Time `xml:"createdAt"` + IsDeleted bool `xml:"isDeleted"` + IsSpam bool `xml:"isSpam"` + Author disqusAuthor `xml:"author"` +} + +type disqusXML struct { + XMLName xml.Name `xml:"disqus"` + Threads []disqusThread `xml:"thread"` + Posts []disqusPost `xml:"post"` +} + +func domainImportDisqus(domain string, url string) (int, error) { + if domain == "" || url == "" { + return 0, errorMissingField + } + + // TODO: make sure this is from disqus.com + resp, err := http.Get(url) + if err != nil { + logger.Errorf("cannot get url: %v", err) + return 0, errorCannotDownloadDisqus + } + + defer resp.Body.Close() + + zr, err := gzip.NewReader(resp.Body) + if err != nil { + logger.Errorf("cannot create gzip reader: %v", err) + return 0, errorInternal + } + + contents, err := ioutil.ReadAll(zr) + if err != nil { + logger.Errorf("cannot read gzip contents uncompressed: %v", err) + return 0, errorInternal + } + + x := disqusXML{} + err = xml.Unmarshal(contents, &x) + if err != nil { + logger.Errorf("cannot unmarshal XML: %v", err) + return 0, errorInternal + } + + // Map Disqus thread IDs to threads. + threads := make(map[string]disqusThread) + for _, thread := range x.Threads { + threads[thread.Id] = thread + } + + // Map Disqus emails to commenterHex (if not available, create a new one + // with a random password that can be reset later). + commenterHex := make(map[string]string) + for _, post := range x.Posts { + if post.IsDeleted || post.IsSpam { + continue + } + + if _, ok := commenterHex[post.Author.Email]; ok { + continue + } + + c, err := commenterGetByEmail("commento", post.Author.Email) + if err != nil && err != errorNoSuchCommenter { + logger.Errorf("cannot get commenter by email: %v", err) + return 0, errorInternal + } + + if err == nil { + commenterHex[post.Author.Email] = c.CommenterHex + continue + } + + randomPassword, err := randomHex(32) + if err != nil { + logger.Errorf("cannot generate random password for new commenter: %v", err) + return 0, errorInternal + } + + commenterHex[post.Author.Email], err = commenterNew(post.Author.Email, post.Author.Name, "undefined", "undefined", "commento", randomPassword) + if err != nil { + return 0, err + } + } + + // For each Disqus post, create a Commento comment. Attempt to convert the + // HTML to markdown. + numImported := 0 + disqusIdMap := make(map[string]string) + for _, post := range x.Posts { + if post.IsDeleted || post.IsSpam { + continue + } + + parentHex := "root" + if val, ok := disqusIdMap[post.ParentId.Id]; ok { + parentHex = val + } + + // TODO: restrict the list of tags to just the basics: , , , + // Especially remove (convert it to ). + commentHex, err := commentNew( + commenterHex[post.Author.Email], + domain, + stripPath(threads[post.ThreadId.Id].URL), + parentHex, + html2md.Convert(post.Message), + "approved", + post.CreationDate) + if err != nil { + return numImported, err + } + + disqusIdMap[post.PostId.Id] = commentHex + numImported += 1 + } + + return numImported, nil +} + +func domainImportDisqusHandler(w http.ResponseWriter, r *http.Request) { + type request struct { + Session *string `json:"session"` + Domain *string `json:"domain"` + URL *string `json:"url"` + } + + var x request + if err := unmarshalBody(r, &x); err != nil { + writeBody(w, response{"success": false, "message": err.Error()}) + return + } + + o, err := ownerGetBySession(*x.Session) + if err != nil { + writeBody(w, response{"success": false, "message": err.Error()}) + return + } + + domain := stripDomain(*x.Domain) + isOwner, err := domainOwnershipVerify(o.OwnerHex, domain) + if err != nil { + writeBody(w, response{"success": false, "message": err.Error()}) + return + } + + if !isOwner { + writeBody(w, response{"success": false, "message": errorNotAuthorised.Error()}) + return + } + + numImported, err := domainImportDisqus(domain, *x.URL) + if err != nil { + writeBody(w, response{"success": false, "message": err.Error()}) + return + } + + writeBody(w, response{"success": true, "numImported": numImported}) +} diff --git a/api/errors.go b/api/errors.go index 2240a2d..99f9d06 100644 --- a/api/errors.go +++ b/api/errors.go @@ -38,3 +38,4 @@ var errorCannotReadResponse = errors.New("Cannot read response.") var errorNotModerator = errors.New("You need to be a moderator to do that.") var errorNotADirectory = errors.New("The given path is not a directory.") var errorGzip = errors.New("Cannot GZip content.") +var errorCannotDownloadDisqus = errors.New("We could not download your Disqus export file.") diff --git a/api/router_api.go b/api/router_api.go index 57c12af..5d96ec2 100644 --- a/api/router_api.go +++ b/api/router_api.go @@ -19,6 +19,7 @@ func initAPIRouter(router *mux.Router) error { router.HandleFunc("/api/domain/moderator/new", domainModeratorNewHandler).Methods("POST") router.HandleFunc("/api/domain/moderator/delete", domainModeratorDeleteHandler).Methods("POST") router.HandleFunc("/api/domain/statistics", domainStatisticsHandler).Methods("POST") + router.HandleFunc("/api/domain/import/disqus", domainImportDisqusHandler).Methods("POST") router.HandleFunc("/api/commenter/session/new", commenterSessionNewHandler).Methods("GET") router.HandleFunc("/api/commenter/new", commenterNewHandler).Methods("POST") diff --git a/frontend/dashboard.html b/frontend/dashboard.html index bf245ba..9d49c96 100644 --- a/frontend/dashboard.html +++ b/frontend/dashboard.html @@ -246,8 +246,8 @@
- - + +
diff --git a/frontend/js/dashboard-import.js b/frontend/js/dashboard-import.js index c7010ef..8bdaf07 100644 --- a/frontend/js/dashboard-import.js +++ b/frontend/js/dashboard-import.js @@ -18,7 +18,7 @@ } global.buttonDisable("#disqus-import-button"); - global.post(global.commento_origin + "/api/import/disqus", json, function(resp) { + global.post(global.commento_origin + "/api/domain/import/disqus", json, function(resp) { global.buttonEnable("#disqus-import-button"); if (!resp.success) { @@ -26,6 +26,8 @@ return; } + $("#disqus-import-button").hide(); + globalOKShow("Imported " + resp.numImported + " comments!"); }); }