api: add disqus endpoint

api/domain_import_disqus.go

@@ -0,0 +1,208 @@
+package main
+
+import (
+	"time"
+	"io/ioutil"
+	"net/http"
+	"compress/gzip"
+	"encoding/xml"
+	// "github.com/grokify/html-strip-tags-go"
+	"github.com/lunny/html2md"
+)
+
+type disqusThread struct {
+	XMLName xml.Name `xml:"thread"`
+	Id string `xml:"http://disqus.com/disqus-internals id,attr"`
+	URL string `xml:"link"`
+	Name string `xml:"name"`
+}
+
+type disqusAuthor struct {
+	XMLName xml.Name `xml:"author"`
+	IsAnonymous bool `xml:"isAnonymous"`
+	Name string `xml:"name"`
+	Email string `xml:"email"`
+}
+
+type disqusThreadId struct {
+	XMLName xml.Name `xml:"thread"`
+	Id      string   `xml:"http://disqus.com/disqus-internals id,attr"`
+}
+
+type disqusParentId struct {
+	XMLName xml.Name `xml:"parent"`
+	Id      string   `xml:"http://disqus.com/disqus-internals id,attr"`
+}
+
+type disqusPostId struct {
+	XMLName xml.Name `xml:"post"`
+	Id      string   `xml:"http://disqus.com/disqus-internals id,attr"`
+}
+
+type disqusPost struct {
+	XMLName xml.Name `xml:"post"`
+	Id string `xml:"http://disqus.com/disqus-internals id,attr"`
+	ThreadId     disqusThreadId `xml:"thread"`
+	ParentId disqusParentId `xml:"parent"`
+	PostId disqusPostId `xml:"post"`
+	Message string `xml:"message"`
+	CreationDate time.Time `xml:"createdAt"`
+	IsDeleted bool `xml:"isDeleted"`
+	IsSpam bool `xml:"isSpam"`
+	Author disqusAuthor `xml:"author"`
+}
+
+type disqusXML struct {
+	XMLName xml.Name `xml:"disqus"`
+	Threads []disqusThread `xml:"thread"`
+	Posts []disqusPost `xml:"post"`
+}
+
+func domainImportDisqus(domain string, url string) (int, error) {
+	if domain == "" || url == "" {
+		return 0, errorMissingField
+	}
+
+	// TODO: make sure this is from disqus.com
+	resp, err := http.Get(url)
+	if err != nil {
+		logger.Errorf("cannot get url: %v", err)
+		return 0, errorCannotDownloadDisqus
+	}
+
+	defer resp.Body.Close()
+
+	zr, err := gzip.NewReader(resp.Body)
+	if err != nil {
+		logger.Errorf("cannot create gzip reader: %v", err)
+		return 0, errorInternal
+	}
+
+	contents, err := ioutil.ReadAll(zr)
+	if err != nil {
+		logger.Errorf("cannot read gzip contents uncompressed: %v", err)
+		return 0, errorInternal
+	}
+
+	x := disqusXML{}
+	err = xml.Unmarshal(contents, &x)
+	if err != nil {
+		logger.Errorf("cannot unmarshal XML: %v", err)
+		return 0, errorInternal
+	}
+
+	// Map Disqus thread IDs to threads.
+	threads := make(map[string]disqusThread)
+	for _, thread := range x.Threads {
+		threads[thread.Id] = thread
+	}
+
+	// Map Disqus emails to commenterHex (if not available, create a new one
+	// with a random password that can be reset later).
+	commenterHex := make(map[string]string)
+	for _, post := range x.Posts {
+		if post.IsDeleted || post.IsSpam {
+			continue
+		}
+
+		if _, ok := commenterHex[post.Author.Email]; ok {
+			continue
+		}
+
+		c, err := commenterGetByEmail("commento", post.Author.Email)
+		if err != nil && err != errorNoSuchCommenter {
+			logger.Errorf("cannot get commenter by email: %v", err)
+			return 0, errorInternal
+		}
+
+		if err == nil {
+			commenterHex[post.Author.Email] = c.CommenterHex
+			continue
+		}
+
+		randomPassword, err := randomHex(32)
+		if err != nil {
+			logger.Errorf("cannot generate random password for new commenter: %v", err)
+			return 0, errorInternal
+		}
+
+		commenterHex[post.Author.Email], err = commenterNew(post.Author.Email, post.Author.Name, "undefined", "undefined", "commento", randomPassword)
+		if err != nil {
+			return 0, err
+		}
+	}
+
+	// For each Disqus post, create a Commento comment. Attempt to convert the
+	// HTML to markdown.
+	numImported := 0
+	disqusIdMap := make(map[string]string)
+	for _, post := range x.Posts {
+		if post.IsDeleted || post.IsSpam {
+			continue
+		}
+
+		parentHex := "root"
+		if val, ok := disqusIdMap[post.ParentId.Id]; ok {
+			parentHex = val
+		}
+
+		// TODO: restrict the list of tags to just the basics: <a>, <b>, <i>, <code>
+		// Especially remove <img> (convert it to <a>).
+		commentHex, err := commentNew(
+			commenterHex[post.Author.Email],
+			domain,
+			stripPath(threads[post.ThreadId.Id].URL),
+			parentHex,
+			html2md.Convert(post.Message),
+			"approved",
+			post.CreationDate)
+		if err != nil {
+			return numImported, err
+		}
+
+		disqusIdMap[post.PostId.Id] = commentHex
+		numImported += 1
+	}
+
+	return numImported, nil
+}
+
+func domainImportDisqusHandler(w http.ResponseWriter, r *http.Request) {
+	type request struct {
+		Session *string `json:"session"`
+		Domain *string `json:"domain"`
+		URL *string `json:"url"`
+	}
+
+	var x request
+	if err := unmarshalBody(r, &x); err != nil {
+		writeBody(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	o, err := ownerGetBySession(*x.Session)
+	if err != nil {
+		writeBody(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	domain := stripDomain(*x.Domain)
+	isOwner, err := domainOwnershipVerify(o.OwnerHex, domain)
+	if err != nil {
+		writeBody(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	if !isOwner {
+		writeBody(w, response{"success": false, "message": errorNotAuthorised.Error()})
+		return
+	}
+
+	numImported, err := domainImportDisqus(domain, *x.URL)
+	if err != nil {
+		writeBody(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	writeBody(w, response{"success": true, "numImported": numImported})
+}

api/errors.go

@@ -38,3 +38,4 @@ var errorCannotReadResponse = errors.New("Cannot read response.")
 var errorNotModerator = errors.New("You need to be a moderator to do that.")
 var errorNotADirectory = errors.New("The given path is not a directory.")
 var errorGzip = errors.New("Cannot GZip content.")
+var errorCannotDownloadDisqus = errors.New("We could not download your Disqus export file.")

api/router_api.go

@@ -19,6 +19,7 @@ func initAPIRouter(router *mux.Router) error {
 	router.HandleFunc("/api/domain/moderator/new", domainModeratorNewHandler).Methods("POST")
 	router.HandleFunc("/api/domain/moderator/delete", domainModeratorDeleteHandler).Methods("POST")
 	router.HandleFunc("/api/domain/statistics", domainStatisticsHandler).Methods("POST")
+	router.HandleFunc("/api/domain/import/disqus", domainImportDisqusHandler).Methods("POST")
 
 	router.HandleFunc("/api/commenter/session/new", commenterSessionNewHandler).Methods("GET")
 	router.HandleFunc("/api/commenter/new", commenterNewHandler).Methods("POST")

frontend/dashboard.html

@@ -246,8 +246,8 @@
 
                         <div class="commento-email-container">
                           <div class="commento-email">
-                            <input class="commento-input" type="text" id="new-mod" placeholder="https://media.disqus.com/uploads/...">
+                            <input class="commento-input" type="text" id="disqus-url" placeholder="https://media.disqus.com/uploads/...">
-                            <button class="commento-email-button">Import</button>
+                            <button id="disqus-import-button" class="commento-email-button" onclick="importDisqus()">Import</button>
                           </div>
                         </div>
                         <div class="subtext-container">

frontend/js/dashboard-import.js

@@ -18,7 +18,7 @@
     }
 
     global.buttonDisable("#disqus-import-button");
-    global.post(global.commento_origin + "/api/import/disqus", json, function(resp) {
+    global.post(global.commento_origin + "/api/domain/import/disqus", json, function(resp) {
       global.buttonEnable("#disqus-import-button");
 
       if (!resp.success) {
@@ -26,6 +26,8 @@
         return;
       }
 
+      $("#disqus-import-button").hide();
+
       globalOKShow("Imported " + resp.numImported + " comments!");
     });
   }