everywhere: add option to export data

api/cron_domain_export_cleanup.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"time"
+)
+
+func domainExportCleanupBegin() error {
+	go func() {
+		for {
+			statement := `
+				DELETE FROM exports
+				WHERE creationDate < $1;
+			`
+			_, err := db.Exec(statement, time.Now().UTC().AddDate(0, -7, 0))
+			if err != nil {
+				logger.Errorf("error cleaning up export rows: %v", err)
+				return
+			}
+
+			time.Sleep(2 * time.Hour)
+		}
+	}()
+
+	return nil
+}

api/domain_export.go

@@ -0,0 +1,146 @@
+package main
+
+import (
+	"net/http"
+	"encoding/json"
+	"time"
+)
+
+func domainExportBeginError(email string, toName string, domain string, err error) {
+	// we're not using err at the moment because it's all errorInternal
+	if err2 := smtpDomainExportError(email, toName, domain); err2 != nil {
+		logger.Errorf("cannot send domain export error email for %s: %v", domain, err2)
+		return
+	}
+}
+
+func domainExportBegin(email string, toName string, domain string) {
+	type dataExport struct {
+		Version int `json:"version"`
+		Comments []comment `json:"comments"`
+		Commenters []commenter `json:"commenters"`
+	}
+
+	e := dataExport{Version: 1, Comments: []comment{}, Commenters: []commenter{}}
+
+	statement := `
+		SELECT commentHex, domain, path, commenterHex, markdown, parentHex, score, state, creationDate
+		FROM comments
+		WHERE domain = $1;
+	`
+	rows1, err := db.Query(statement, domain)
+	if err != nil {
+		logger.Errorf("cannot select comments while exporting %s: %v", domain, err)
+		domainExportBeginError(email, toName, domain, errorInternal)
+		return
+	}
+	defer rows1.Close()
+
+	for rows1.Next() {
+		c := comment{}
+		if err = rows1.Scan(&c.CommentHex, &c.Domain, &c.Path, &c.CommenterHex, &c.Markdown, &c.ParentHex, &c.Score, &c.State, &c.CreationDate); err != nil {
+			logger.Errorf("cannot scan comment while exporting %s: %v", domain, err)
+			domainExportBeginError(email, toName, domain, errorInternal)
+			return
+		}
+
+		e.Comments = append(e.Comments, c)
+	}
+
+	statement = `
+		SELECT commenters.commenterHex, commenters.email, commenters.name, commenters.link, commenters.photo, commenters.provider, commenters.joinDate
+		FROM commenters, comments
+		WHERE comments.domain = $1 AND commenters.commenterHex = comments.commenterHex;
+	`
+	rows2, err := db.Query(statement, domain)
+	if err != nil {
+		logger.Errorf("cannot select commenters while exporting %s: %v", domain, err)
+		domainExportBeginError(email, toName, domain, errorInternal)
+		return
+	}
+	defer rows2.Close()
+
+	for rows2.Next() {
+		c := commenter{}
+		if err := rows2.Scan(&c.CommenterHex, &c.Email, &c.Name, &c.Link, &c.Photo, &c.Provider, &c.JoinDate); err != nil {
+			logger.Errorf("cannot scan commenter while exporting %s: %v", domain, err)
+			domainExportBeginError(email, toName, domain, errorInternal)
+			return
+		}
+
+		e.Commenters = append(e.Commenters, c)
+	}
+
+	je, err := json.Marshal(e)
+	if err != nil {
+		logger.Errorf("cannot marshall JSON while exporting %s: %v", domain, err)
+		domainExportBeginError(email, toName, domain, errorInternal)
+		return
+	}
+
+	gje, err := gzipStatic(je)
+	if err != nil {
+		logger.Errorf("cannot gzip JSON while exporting %s: %v", domain, err)
+		domainExportBeginError(email, toName, domain, errorInternal)
+		return
+	}
+
+	exportHex, err := randomHex(32)
+	if err != nil {
+		logger.Errorf("cannot generate exportHex while exporting %s: %v", domain, err)
+		domainExportBeginError(email, toName, domain, errorInternal)
+		return
+	}
+
+	statement = `
+		INSERT INTO
+		exports (exportHex, binData, domain, creationDate)
+		VALUES  ($1,        $2,      $3    , $4          );
+	`
+	_, err = db.Exec(statement, exportHex, gje, domain, time.Now().UTC())
+	if err != nil {
+		logger.Errorf("error inserting expiry binary data while exporting %s: %v", domain, err)
+		domainExportBeginError(email, toName, domain, errorInternal)
+		return
+	}
+
+	err = smtpDomainExport(email, toName, domain, exportHex)
+	if err != nil {
+		logger.Errorf("error sending data export email for %s: %v", domain, err)
+		return
+	}
+}
+
+func domainExportBeginHandler(w http.ResponseWriter, r *http.Request) {
+	type request struct {
+		OwnerToken *string `json:"ownerToken"`
+		Domain *string `json:"domain"`
+	}
+
+	var x request
+	if err := bodyUnmarshal(r, &x); err != nil {
+		bodyMarshal(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	o, err := ownerGetByOwnerToken(*x.OwnerToken)
+	if err != nil {
+		bodyMarshal(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	isOwner, err := domainOwnershipVerify(o.OwnerHex, *x.Domain)
+	if err != nil {
+		bodyMarshal(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	if !isOwner {
+		bodyMarshal(w, response{"success": false, "message": errorNotAuthorised.Error()})
+		return
+	}
+
+	go domainExportBegin(o.Email, o.Name, *x.Domain);
+
+	bodyMarshal(w, response{"success": true})
+}

api/domain_export_download.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"net/http"
+	"fmt"
+	"time"
+)
+
+func domainExportDownloadHandler(w http.ResponseWriter, r *http.Request) {
+	exportHex := r.FormValue("exportHex")
+	if exportHex == "" {
+		fmt.Fprintf(w, "Error: empty exportHex\n")
+		return
+	}
+
+	statement := `
+		SELECT domain, binData, creationDate
+		FROM exports
+		WHERE exportHex = $1;
+	`
+	row := db.QueryRow(statement, exportHex)
+
+	var domain string
+	var binData []byte
+	var creationDate time.Time
+	if err := row.Scan(&domain, &binData, &creationDate); err != nil {
+		fmt.Fprintf(w, "Error: that exportHex does not exist\n")
+	}
+
+	w.Header().Set("Content-Disposition", fmt.Sprintf(`inline; filename="%s-%v.gz"`, domain, creationDate.Unix()))
+	w.Header().Set("Content-Encoding", "gzip")
+	w.Write(binData)
+}

api/main.go

@@ -11,6 +11,7 @@ func main() {
 	exitIfError(markdownRendererCreate())
 	exitIfError(sigintCleanupSetup())
 	exitIfError(versionCheckStart())
+	exitIfError(domainExportCleanupBegin())
 
 	exitIfError(routesServe())
 }

api/router_api.go

@@ -20,6 +20,8 @@ func apiRouterInit(router *mux.Router) error {
 	router.HandleFunc("/api/domain/moderator/delete", domainModeratorDeleteHandler).Methods("POST")
 	router.HandleFunc("/api/domain/statistics", domainStatisticsHandler).Methods("POST")
 	router.HandleFunc("/api/domain/import/disqus", domainImportDisqusHandler).Methods("POST")
+	router.HandleFunc("/api/domain/export/begin", domainExportBeginHandler).Methods("POST")
+	router.HandleFunc("/api/domain/export/download", domainExportDownloadHandler).Methods("GET")
 
 	router.HandleFunc("/api/commenter/token/new", commenterTokenNewHandler).Methods("GET")
 	router.HandleFunc("/api/commenter/new", commenterNewHandler).Methods("POST")

api/smtp_domain_export.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"bytes"
+	"net/smtp"
+	"os"
+)
+
+type domainExportPlugs struct {
+	Origin     string
+	Domain string
+	ExportHex string
+}
+
+func smtpDomainExport(to string, toName string, domain string, exportHex string) error {
+	var header bytes.Buffer
+	headerTemplate.Execute(&header, &headerPlugs{FromAddress: os.Getenv("SMTP_FROM_ADDRESS"), ToAddress: to, ToName: toName, Subject: "Commento Data Export"})
+
+	var body bytes.Buffer
+	templates["domain-export"].Execute(&body, &domainExportPlugs{Origin: os.Getenv("ORIGIN"), ExportHex: exportHex})
+
+	err := smtp.SendMail(os.Getenv("SMTP_HOST")+":"+os.Getenv("SMTP_PORT"), smtpAuth, os.Getenv("SMTP_FROM_ADDRESS"), []string{to}, concat(header, body))
+	if err != nil {
+		logger.Errorf("cannot send data export email: %v", err)
+		return errorCannotSendEmail
+	}
+
+	return nil
+}

api/smtp_domain_export_error.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"bytes"
+	"net/smtp"
+	"os"
+)
+
+type domainExportErrorPlugs struct {
+	Origin     string
+	Domain string
+}
+
+func smtpDomainExportError(to string, toName string, domain string) error {
+	var header bytes.Buffer
+	headerTemplate.Execute(&header, &headerPlugs{FromAddress: os.Getenv("SMTP_FROM_ADDRESS"), ToAddress: to, ToName: toName, Subject: "Commento Data Export"})
+
+	var body bytes.Buffer
+	templates["data-export-error"].Execute(&body, &domainExportPlugs{Origin: os.Getenv("ORIGIN")})
+
+	err := smtp.SendMail(os.Getenv("SMTP_HOST")+":"+os.Getenv("SMTP_PORT"), smtpAuth, os.Getenv("SMTP_FROM_ADDRESS"), []string{to}, concat(header, body))
+	if err != nil {
+		logger.Errorf("cannot send data export error email: %v", err)
+		return errorCannotSendEmail
+	}
+
+	return nil
+}

api/smtp_templates.go

@@ -31,7 +31,7 @@ Subject: {{.Subject}}
 		return errorMalformedTemplate
 	}
 
-	names := []string{"confirm-hex", "reset-hex"}
+	names := []string{"confirm-hex", "reset-hex", "domain-export", "domain-export-error"}
 
 	templates = make(map[string]*template.Template)
 

db/20190131002240-export.sql

@@ -0,0 +1,8 @@
+-- add export feature
+
+CREATE TABLE IF NOT EXISTS exports (
+  exportHex TEXT NOT NULL UNIQUE PRIMARY KEY,
+  binData BYTEA NOT NULL,
+  domain TEXT NOT NULL,
+  creationDate TIMESTAMP NOT NULL
+);

frontend/dashboard.html

@@ -82,27 +82,27 @@
       <!-- Installation -->
       <div id="installation-view" class="view hidden">
         <div class="view-inside">
-          <div class="large-view">
+          <div class="mid-view">
             <div class="tabs-container">
               <div class="tab">
                 <ul class="tabs">
-                  <li class="tab-link original current" data-tab="install-tab-1">Universal Snippet</li>
+                  <li class="tab-link original current" data-tab="installation-tab-1">Universal Snippet</li>
                 </ul>
 
-                <div id="install-tab-1" class="content original current">
+                <div id="installation-tab-1" class="content original current">
-                  <div class="import-text">
+                  <div class="normal-text">
                     Copy the following piece of HTML code and paste it where you'd like Commento to load.
                   </div>
 
                   <pre><code id="code-div" class="html"></code></pre>
 
-                  <div class="import-text">
+                  <div class="normal-text">
                     And that's it. All your settings, themes, and comments would be automagically loaded. Commento is mobile-responsive too, as it simply fills the container it is put in.
                   </div>
                   
                   <br>
 
-                  <div class="import-text">
+                  <div class="normal-text">
                     Read the Commento documentation <a href="https://docs.commento.io/configuration/">on configuration</a>.
                   </div>
                 </div>
@@ -120,7 +120,7 @@
               Analytics
             </div>
 
-            <div class="import-text">
+            <div class="normal-text">
               Anonymous statistics such as monthly pageviews and number of comments
             </div>
 
@@ -154,7 +154,7 @@
       <!-- moderation -->
       <div id="moderation-view" class="view hidden">
         <div class="view-inside">
-          <div class="small-view small-mid-view">
+          <div class="mid-view">
             <div class="tabs-container">
               <div class="tab">
                 <ul class="tabs">
@@ -231,20 +231,42 @@
       <!-- Configure Domain -->
       <div id="general-view" class="view hidden">
         <div class="view-inside">
-          <div class="small-mid-view">
+          <div class="mid-view">
-            <div class="center center-title">
+            <div class="tabs-container">
-              Configure Domain
+              <div class="tab">
-            </div>
+                <ul class="tabs">
-            <div class="box">
+                  <li class="tab-link original current" data-tab="configure-tab-1">General</li>
-              <div class="row">
+                  <li class="tab-link" data-tab="configure-tab-2">Email Settings</li>
-                <div class="label">Website Name</div>
+                  <li class="tab-link" data-tab="configure-tab-3">Export Data</li>
-                <input class="input gray-input" id="cur-domain-name" type="text" :placeholder="domains[cd].origName" v-model="domains[cd].name">
+                </ul>
-              </div>
 
-              <div id="new-domain-error" class="modal-error-box"></div>
+                <div id="configure-tab-1" class="content original current">
-            </div>
+                  <div class="box">
-            <div class="center">
+                    <div class="row">
-              <button id="save-general-button" onclick="window.commento.generalSaveHandler()" class="button">Save Changes</button>
+                      <div class="label">Website Name</div>
+                      <input class="input gray-input" id="cur-domain-name" type="text" :placeholder="domains[cd].origName" v-model="domains[cd].name">
+                    </div>
+                  </div>
+                  <div class="center">
+                    <button id="save-general-button" onclick="window.commento.generalSaveHandler()" class="button">Save Changes</button>
+                  </div>
+                </div>
+
+                <div id="configure-tab-2" class="content">
+                </div>
+
+                <div id="configure-tab-3" class="content">
+                  <div class="normal-text">
+                    You can export an archive of this domain's data (which includes all comments and commenters) in the JSON format. To initiate and queue an archive request, click the button below. You will receive an email containing the archive once it's ready.<br><br>
+
+                    Please note that this requires valid SMTP settings in order to send emails.<br><br>
+
+                    <div class="center">
+                      <button id="domain-export-button" onclick="window.commento.domainExportBegin()" class="button">Initiate Data Export</button>
+                    </div>
+                  </div>
+                </div>
+              </div>
             </div>
           </div>
         </div>
@@ -253,7 +275,7 @@
       <!-- Import Comments -->
       <div id="import-view" class="view hidden">
         <div class="view-inside">
-          <div class="large-view">
+          <div class="mid-view">
             <div class="tabs-container">
               <div class="tab">
                 <ul class="tabs">
@@ -261,7 +283,7 @@
                 </ul>
 
                 <div id="install-tab-1" class="content original current">
-                  <div class="import-text">
+                  <div class="normal-text">
                     If you're currently using Disqus, you can import all comments into Commento:
                     <ul>
                       <li>

frontend/gulpfile.js

@@ -66,6 +66,7 @@ const jsCompileMap = {
     "js/dashboard-statistics.js",
     "js/dashboard-import.js",
     "js/dashboard-danger.js",
+    "js/dashboard-export.js",
   ],
   "js/logout.js": [
     "js/constants.js",

frontend/js/dashboard-export.js

@@ -0,0 +1,26 @@
+(function (global, document) {
+  "use strict";
+
+  (document);
+
+  global.domainExportBegin = function() {
+    var data = global.dashboard.$data;
+
+    var json = {
+      "ownerToken": global.cookieGet("commentoOwnerToken"),
+      "domain": data.domains[data.cd].domain,
+    }
+
+    global.buttonDisable("#domain-export-button");
+    global.post(global.origin + "/api/domain/export/begin", json, function(resp) {
+      global.buttonEnable("#domain-export-button");
+      if (!resp.success) {
+        global.globalErrorShow(resp.message);
+        return;
+      }
+
+      global.globalOKShow("Data export operation has been successfully queued. You will receive an email.");
+    });
+  };
+
+} (window.commento, document));

frontend/js/dashboard.js

@@ -30,8 +30,8 @@
       },
       {
         "id": "general",
-        "text": "General Settings",
+        "text": "General",
-        "meaning": "Names, emails, and general settings",
+        "meaning": "Email settings, data export",
         "selected": false,
         "open": global.generalOpen,
       },

frontend/sass/dashboard-main.scss

@@ -375,10 +375,10 @@ body {
   }
 }
 
-.import-text {
+.normal-text {
-  font-size: 15px;
+  font-size: 14px;
   color: $gray-7;
-  line-height: 25px;
+  line-height: 22px;
 
   a {
     color: $blue-6;
@@ -391,7 +391,7 @@ body {
 
   li {
     position: relative;
-    margin-bottom: 10px;
+    margin-bottom: 5px;
   }
 
   li::before {

templates/domain-export-error.txt

@@ -0,0 +1,3 @@
+You recently requested a data export of your Commento domain {{.Domain}}. An
+error was encountered while processing the request. Please contact support to
+resolve this issue.

templates/domain-export.txt

@@ -0,0 +1,7 @@
+You recently requested a data export of your Commento domain {{.Domain}}. You
+can download a GZipped archive of a JSON export of all the comments and
+commenters associated with the domain here:
+
+{{.Origin}}/api/domain/export/download?exportHex={{.ExportHex}}
+
+The archive will be available for download for 7 days.