everywhere: add option to export data

api/cron_domain_export_cleanup.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"time"
+)
+
+func domainExportCleanupBegin() error {
+	go func() {
+		for {
+			statement := `
+				DELETE FROM exports
+				WHERE creationDate < $1;
+			`
+			_, err := db.Exec(statement, time.Now().UTC().AddDate(0, -7, 0))
+			if err != nil {
+				logger.Errorf("error cleaning up export rows: %v", err)
+				return
+			}
+
+			time.Sleep(2 * time.Hour)
+		}
+	}()
+
+	return nil
+}

api/domain_export.go

@@ -0,0 +1,146 @@
+package main
+
+import (
+	"net/http"
+	"encoding/json"
+	"time"
+)
+
+func domainExportBeginError(email string, toName string, domain string, err error) {
+	// we're not using err at the moment because it's all errorInternal
+	if err2 := smtpDomainExportError(email, toName, domain); err2 != nil {
+		logger.Errorf("cannot send domain export error email for %s: %v", domain, err2)
+		return
+	}
+}
+
+func domainExportBegin(email string, toName string, domain string) {
+	type dataExport struct {
+		Version int `json:"version"`
+		Comments []comment `json:"comments"`
+		Commenters []commenter `json:"commenters"`
+	}
+
+	e := dataExport{Version: 1, Comments: []comment{}, Commenters: []commenter{}}
+
+	statement := `
+		SELECT commentHex, domain, path, commenterHex, markdown, parentHex, score, state, creationDate
+		FROM comments
+		WHERE domain = $1;
+	`
+	rows1, err := db.Query(statement, domain)
+	if err != nil {
+		logger.Errorf("cannot select comments while exporting %s: %v", domain, err)
+		domainExportBeginError(email, toName, domain, errorInternal)
+		return
+	}
+	defer rows1.Close()
+
+	for rows1.Next() {
+		c := comment{}
+		if err = rows1.Scan(&c.CommentHex, &c.Domain, &c.Path, &c.CommenterHex, &c.Markdown, &c.ParentHex, &c.Score, &c.State, &c.CreationDate); err != nil {
+			logger.Errorf("cannot scan comment while exporting %s: %v", domain, err)
+			domainExportBeginError(email, toName, domain, errorInternal)
+			return
+		}
+
+		e.Comments = append(e.Comments, c)
+	}
+
+	statement = `
+		SELECT commenters.commenterHex, commenters.email, commenters.name, commenters.link, commenters.photo, commenters.provider, commenters.joinDate
+		FROM commenters, comments
+		WHERE comments.domain = $1 AND commenters.commenterHex = comments.commenterHex;
+	`
+	rows2, err := db.Query(statement, domain)
+	if err != nil {
+		logger.Errorf("cannot select commenters while exporting %s: %v", domain, err)
+		domainExportBeginError(email, toName, domain, errorInternal)
+		return
+	}
+	defer rows2.Close()
+
+	for rows2.Next() {
+		c := commenter{}
+		if err := rows2.Scan(&c.CommenterHex, &c.Email, &c.Name, &c.Link, &c.Photo, &c.Provider, &c.JoinDate); err != nil {
+			logger.Errorf("cannot scan commenter while exporting %s: %v", domain, err)
+			domainExportBeginError(email, toName, domain, errorInternal)
+			return
+		}
+
+		e.Commenters = append(e.Commenters, c)
+	}
+
+	je, err := json.Marshal(e)
+	if err != nil {
+		logger.Errorf("cannot marshall JSON while exporting %s: %v", domain, err)
+		domainExportBeginError(email, toName, domain, errorInternal)
+		return
+	}
+
+	gje, err := gzipStatic(je)
+	if err != nil {
+		logger.Errorf("cannot gzip JSON while exporting %s: %v", domain, err)
+		domainExportBeginError(email, toName, domain, errorInternal)
+		return
+	}
+
+	exportHex, err := randomHex(32)
+	if err != nil {
+		logger.Errorf("cannot generate exportHex while exporting %s: %v", domain, err)
+		domainExportBeginError(email, toName, domain, errorInternal)
+		return
+	}
+
+	statement = `
+		INSERT INTO
+		exports (exportHex, binData, domain, creationDate)
+		VALUES  ($1,        $2,      $3    , $4          );
+	`
+	_, err = db.Exec(statement, exportHex, gje, domain, time.Now().UTC())
+	if err != nil {
+		logger.Errorf("error inserting expiry binary data while exporting %s: %v", domain, err)
+		domainExportBeginError(email, toName, domain, errorInternal)
+		return
+	}
+
+	err = smtpDomainExport(email, toName, domain, exportHex)
+	if err != nil {
+		logger.Errorf("error sending data export email for %s: %v", domain, err)
+		return
+	}
+}
+
+func domainExportBeginHandler(w http.ResponseWriter, r *http.Request) {
+	type request struct {
+		OwnerToken *string `json:"ownerToken"`
+		Domain *string `json:"domain"`
+	}
+
+	var x request
+	if err := bodyUnmarshal(r, &x); err != nil {
+		bodyMarshal(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	o, err := ownerGetByOwnerToken(*x.OwnerToken)
+	if err != nil {
+		bodyMarshal(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	isOwner, err := domainOwnershipVerify(o.OwnerHex, *x.Domain)
+	if err != nil {
+		bodyMarshal(w, response{"success": false, "message": err.Error()})
+		return
+	}
+
+	if !isOwner {
+		bodyMarshal(w, response{"success": false, "message": errorNotAuthorised.Error()})
+		return
+	}
+
+	go domainExportBegin(o.Email, o.Name, *x.Domain);
+
+	bodyMarshal(w, response{"success": true})
+}

api/domain_export_download.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"net/http"
+	"fmt"
+	"time"
+)
+
+func domainExportDownloadHandler(w http.ResponseWriter, r *http.Request) {
+	exportHex := r.FormValue("exportHex")
+	if exportHex == "" {
+		fmt.Fprintf(w, "Error: empty exportHex\n")
+		return
+	}
+
+	statement := `
+		SELECT domain, binData, creationDate
+		FROM exports
+		WHERE exportHex = $1;
+	`
+	row := db.QueryRow(statement, exportHex)
+
+	var domain string
+	var binData []byte
+	var creationDate time.Time
+	if err := row.Scan(&domain, &binData, &creationDate); err != nil {
+		fmt.Fprintf(w, "Error: that exportHex does not exist\n")
+	}
+
+	w.Header().Set("Content-Disposition", fmt.Sprintf(`inline; filename="%s-%v.gz"`, domain, creationDate.Unix()))
+	w.Header().Set("Content-Encoding", "gzip")
+	w.Write(binData)
+}

api/main.go

@@ -11,6 +11,7 @@ func main() {
 	exitIfError(markdownRendererCreate())
 	exitIfError(sigintCleanupSetup())
 	exitIfError(versionCheckStart())
+	exitIfError(domainExportCleanupBegin())
 
 	exitIfError(routesServe())
 }

api/router_api.go

@@ -20,6 +20,8 @@ func apiRouterInit(router *mux.Router) error {
 	router.HandleFunc("/api/domain/moderator/delete", domainModeratorDeleteHandler).Methods("POST")
 	router.HandleFunc("/api/domain/statistics", domainStatisticsHandler).Methods("POST")
 	router.HandleFunc("/api/domain/import/disqus", domainImportDisqusHandler).Methods("POST")
+	router.HandleFunc("/api/domain/export/begin", domainExportBeginHandler).Methods("POST")
+	router.HandleFunc("/api/domain/export/download", domainExportDownloadHandler).Methods("GET")
 
 	router.HandleFunc("/api/commenter/token/new", commenterTokenNewHandler).Methods("GET")
 	router.HandleFunc("/api/commenter/new", commenterNewHandler).Methods("POST")

api/smtp_domain_export.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"bytes"
+	"net/smtp"
+	"os"
+)
+
+type domainExportPlugs struct {
+	Origin     string
+	Domain string
+	ExportHex string
+}
+
+func smtpDomainExport(to string, toName string, domain string, exportHex string) error {
+	var header bytes.Buffer
+	headerTemplate.Execute(&header, &headerPlugs{FromAddress: os.Getenv("SMTP_FROM_ADDRESS"), ToAddress: to, ToName: toName, Subject: "Commento Data Export"})
+
+	var body bytes.Buffer
+	templates["domain-export"].Execute(&body, &domainExportPlugs{Origin: os.Getenv("ORIGIN"), ExportHex: exportHex})
+
+	err := smtp.SendMail(os.Getenv("SMTP_HOST")+":"+os.Getenv("SMTP_PORT"), smtpAuth, os.Getenv("SMTP_FROM_ADDRESS"), []string{to}, concat(header, body))
+	if err != nil {
+		logger.Errorf("cannot send data export email: %v", err)
+		return errorCannotSendEmail
+	}
+
+	return nil
+}

api/smtp_domain_export_error.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"bytes"
+	"net/smtp"
+	"os"
+)
+
+type domainExportErrorPlugs struct {
+	Origin     string
+	Domain string
+}
+
+func smtpDomainExportError(to string, toName string, domain string) error {
+	var header bytes.Buffer
+	headerTemplate.Execute(&header, &headerPlugs{FromAddress: os.Getenv("SMTP_FROM_ADDRESS"), ToAddress: to, ToName: toName, Subject: "Commento Data Export"})
+
+	var body bytes.Buffer
+	templates["data-export-error"].Execute(&body, &domainExportPlugs{Origin: os.Getenv("ORIGIN")})
+
+	err := smtp.SendMail(os.Getenv("SMTP_HOST")+":"+os.Getenv("SMTP_PORT"), smtpAuth, os.Getenv("SMTP_FROM_ADDRESS"), []string{to}, concat(header, body))
+	if err != nil {
+		logger.Errorf("cannot send data export error email: %v", err)
+		return errorCannotSendEmail
+	}
+
+	return nil
+}

api/smtp_templates.go

@@ -31,7 +31,7 @@ Subject: {{.Subject}}
 		return errorMalformedTemplate
 	}
 
-	names := []string{"confirm-hex", "reset-hex"}
+	names := []string{"confirm-hex", "reset-hex", "domain-export", "domain-export-error"}
 
 	templates = make(map[string]*template.Template)