From fff5e5c0e1bbb3a4e926ba39b8d8934542cf4e61 Mon Sep 17 00:00:00 2001 From: Adhityaa Chandrasekar Date: Thu, 31 Jan 2019 02:06:11 -0500 Subject: [PATCH] everywhere: add option to export data --- api/cron_domain_export_cleanup.go | 25 +++++ api/domain_export.go | 146 ++++++++++++++++++++++++++++++ api/domain_export_download.go | 33 +++++++ api/main.go | 1 + api/router_api.go | 2 + api/smtp_domain_export.go | 29 ++++++ api/smtp_domain_export_error.go | 28 ++++++ api/smtp_templates.go | 2 +- db/20190131002240-export.sql | 8 ++ frontend/dashboard.html | 68 +++++++++----- frontend/gulpfile.js | 1 + frontend/js/dashboard-export.js | 26 ++++++ frontend/js/dashboard.js | 4 +- frontend/sass/dashboard-main.scss | 8 +- templates/domain-export-error.txt | 3 + templates/domain-export.txt | 7 ++ 16 files changed, 361 insertions(+), 30 deletions(-) create mode 100644 api/cron_domain_export_cleanup.go create mode 100644 api/domain_export.go create mode 100644 api/domain_export_download.go create mode 100644 api/smtp_domain_export.go create mode 100644 api/smtp_domain_export_error.go create mode 100644 db/20190131002240-export.sql create mode 100644 frontend/js/dashboard-export.js create mode 100644 templates/domain-export-error.txt create mode 100644 templates/domain-export.txt diff --git a/api/cron_domain_export_cleanup.go b/api/cron_domain_export_cleanup.go new file mode 100644 index 0000000..ec19016 --- /dev/null +++ b/api/cron_domain_export_cleanup.go @@ -0,0 +1,25 @@ +package main + +import ( + "time" +) + +func domainExportCleanupBegin() error { + go func() { + for { + statement := ` + DELETE FROM exports + WHERE creationDate < $1; + ` + _, err := db.Exec(statement, time.Now().UTC().AddDate(0, -7, 0)) + if err != nil { + logger.Errorf("error cleaning up export rows: %v", err) + return + } + + time.Sleep(2 * time.Hour) + } + }() + + return nil +} diff --git a/api/domain_export.go b/api/domain_export.go new file mode 100644 index 0000000..7a2e653 --- /dev/null +++ b/api/domain_export.go @@ -0,0 +1,146 @@ +package main + +import ( + "net/http" + "encoding/json" + "time" +) + +func domainExportBeginError(email string, toName string, domain string, err error) { + // we're not using err at the moment because it's all errorInternal + if err2 := smtpDomainExportError(email, toName, domain); err2 != nil { + logger.Errorf("cannot send domain export error email for %s: %v", domain, err2) + return + } +} + +func domainExportBegin(email string, toName string, domain string) { + type dataExport struct { + Version int `json:"version"` + Comments []comment `json:"comments"` + Commenters []commenter `json:"commenters"` + } + + e := dataExport{Version: 1, Comments: []comment{}, Commenters: []commenter{}} + + statement := ` + SELECT commentHex, domain, path, commenterHex, markdown, parentHex, score, state, creationDate + FROM comments + WHERE domain = $1; + ` + rows1, err := db.Query(statement, domain) + if err != nil { + logger.Errorf("cannot select comments while exporting %s: %v", domain, err) + domainExportBeginError(email, toName, domain, errorInternal) + return + } + defer rows1.Close() + + for rows1.Next() { + c := comment{} + if err = rows1.Scan(&c.CommentHex, &c.Domain, &c.Path, &c.CommenterHex, &c.Markdown, &c.ParentHex, &c.Score, &c.State, &c.CreationDate); err != nil { + logger.Errorf("cannot scan comment while exporting %s: %v", domain, err) + domainExportBeginError(email, toName, domain, errorInternal) + return + } + + e.Comments = append(e.Comments, c) + } + + statement = ` + SELECT commenters.commenterHex, commenters.email, commenters.name, commenters.link, commenters.photo, commenters.provider, commenters.joinDate + FROM commenters, comments + WHERE comments.domain = $1 AND commenters.commenterHex = comments.commenterHex; + ` + rows2, err := db.Query(statement, domain) + if err != nil { + logger.Errorf("cannot select commenters while exporting %s: %v", domain, err) + domainExportBeginError(email, toName, domain, errorInternal) + return + } + defer rows2.Close() + + for rows2.Next() { + c := commenter{} + if err := rows2.Scan(&c.CommenterHex, &c.Email, &c.Name, &c.Link, &c.Photo, &c.Provider, &c.JoinDate); err != nil { + logger.Errorf("cannot scan commenter while exporting %s: %v", domain, err) + domainExportBeginError(email, toName, domain, errorInternal) + return + } + + e.Commenters = append(e.Commenters, c) + } + + je, err := json.Marshal(e) + if err != nil { + logger.Errorf("cannot marshall JSON while exporting %s: %v", domain, err) + domainExportBeginError(email, toName, domain, errorInternal) + return + } + + gje, err := gzipStatic(je) + if err != nil { + logger.Errorf("cannot gzip JSON while exporting %s: %v", domain, err) + domainExportBeginError(email, toName, domain, errorInternal) + return + } + + exportHex, err := randomHex(32) + if err != nil { + logger.Errorf("cannot generate exportHex while exporting %s: %v", domain, err) + domainExportBeginError(email, toName, domain, errorInternal) + return + } + + statement = ` + INSERT INTO + exports (exportHex, binData, domain, creationDate) + VALUES ($1, $2, $3 , $4 ); + ` + _, err = db.Exec(statement, exportHex, gje, domain, time.Now().UTC()) + if err != nil { + logger.Errorf("error inserting expiry binary data while exporting %s: %v", domain, err) + domainExportBeginError(email, toName, domain, errorInternal) + return + } + + err = smtpDomainExport(email, toName, domain, exportHex) + if err != nil { + logger.Errorf("error sending data export email for %s: %v", domain, err) + return + } +} + +func domainExportBeginHandler(w http.ResponseWriter, r *http.Request) { + type request struct { + OwnerToken *string `json:"ownerToken"` + Domain *string `json:"domain"` + } + + var x request + if err := bodyUnmarshal(r, &x); err != nil { + bodyMarshal(w, response{"success": false, "message": err.Error()}) + return + } + + o, err := ownerGetByOwnerToken(*x.OwnerToken) + if err != nil { + bodyMarshal(w, response{"success": false, "message": err.Error()}) + return + } + + isOwner, err := domainOwnershipVerify(o.OwnerHex, *x.Domain) + if err != nil { + bodyMarshal(w, response{"success": false, "message": err.Error()}) + return + } + + if !isOwner { + bodyMarshal(w, response{"success": false, "message": errorNotAuthorised.Error()}) + return + } + + go domainExportBegin(o.Email, o.Name, *x.Domain); + + bodyMarshal(w, response{"success": true}) +} diff --git a/api/domain_export_download.go b/api/domain_export_download.go new file mode 100644 index 0000000..a9274ce --- /dev/null +++ b/api/domain_export_download.go @@ -0,0 +1,33 @@ +package main + +import ( + "net/http" + "fmt" + "time" +) + +func domainExportDownloadHandler(w http.ResponseWriter, r *http.Request) { + exportHex := r.FormValue("exportHex") + if exportHex == "" { + fmt.Fprintf(w, "Error: empty exportHex\n") + return + } + + statement := ` + SELECT domain, binData, creationDate + FROM exports + WHERE exportHex = $1; + ` + row := db.QueryRow(statement, exportHex) + + var domain string + var binData []byte + var creationDate time.Time + if err := row.Scan(&domain, &binData, &creationDate); err != nil { + fmt.Fprintf(w, "Error: that exportHex does not exist\n") + } + + w.Header().Set("Content-Disposition", fmt.Sprintf(`inline; filename="%s-%v.gz"`, domain, creationDate.Unix())) + w.Header().Set("Content-Encoding", "gzip") + w.Write(binData) +} diff --git a/api/main.go b/api/main.go index 7d309d2..2aaeef6 100644 --- a/api/main.go +++ b/api/main.go @@ -11,6 +11,7 @@ func main() { exitIfError(markdownRendererCreate()) exitIfError(sigintCleanupSetup()) exitIfError(versionCheckStart()) + exitIfError(domainExportCleanupBegin()) exitIfError(routesServe()) } diff --git a/api/router_api.go b/api/router_api.go index 59bae75..2bbaf18 100644 --- a/api/router_api.go +++ b/api/router_api.go @@ -20,6 +20,8 @@ func apiRouterInit(router *mux.Router) error { router.HandleFunc("/api/domain/moderator/delete", domainModeratorDeleteHandler).Methods("POST") router.HandleFunc("/api/domain/statistics", domainStatisticsHandler).Methods("POST") router.HandleFunc("/api/domain/import/disqus", domainImportDisqusHandler).Methods("POST") + router.HandleFunc("/api/domain/export/begin", domainExportBeginHandler).Methods("POST") + router.HandleFunc("/api/domain/export/download", domainExportDownloadHandler).Methods("GET") router.HandleFunc("/api/commenter/token/new", commenterTokenNewHandler).Methods("GET") router.HandleFunc("/api/commenter/new", commenterNewHandler).Methods("POST") diff --git a/api/smtp_domain_export.go b/api/smtp_domain_export.go new file mode 100644 index 0000000..0b47514 --- /dev/null +++ b/api/smtp_domain_export.go @@ -0,0 +1,29 @@ +package main + +import ( + "bytes" + "net/smtp" + "os" +) + +type domainExportPlugs struct { + Origin string + Domain string + ExportHex string +} + +func smtpDomainExport(to string, toName string, domain string, exportHex string) error { + var header bytes.Buffer + headerTemplate.Execute(&header, &headerPlugs{FromAddress: os.Getenv("SMTP_FROM_ADDRESS"), ToAddress: to, ToName: toName, Subject: "Commento Data Export"}) + + var body bytes.Buffer + templates["domain-export"].Execute(&body, &domainExportPlugs{Origin: os.Getenv("ORIGIN"), ExportHex: exportHex}) + + err := smtp.SendMail(os.Getenv("SMTP_HOST")+":"+os.Getenv("SMTP_PORT"), smtpAuth, os.Getenv("SMTP_FROM_ADDRESS"), []string{to}, concat(header, body)) + if err != nil { + logger.Errorf("cannot send data export email: %v", err) + return errorCannotSendEmail + } + + return nil +} diff --git a/api/smtp_domain_export_error.go b/api/smtp_domain_export_error.go new file mode 100644 index 0000000..50ee137 --- /dev/null +++ b/api/smtp_domain_export_error.go @@ -0,0 +1,28 @@ +package main + +import ( + "bytes" + "net/smtp" + "os" +) + +type domainExportErrorPlugs struct { + Origin string + Domain string +} + +func smtpDomainExportError(to string, toName string, domain string) error { + var header bytes.Buffer + headerTemplate.Execute(&header, &headerPlugs{FromAddress: os.Getenv("SMTP_FROM_ADDRESS"), ToAddress: to, ToName: toName, Subject: "Commento Data Export"}) + + var body bytes.Buffer + templates["data-export-error"].Execute(&body, &domainExportPlugs{Origin: os.Getenv("ORIGIN")}) + + err := smtp.SendMail(os.Getenv("SMTP_HOST")+":"+os.Getenv("SMTP_PORT"), smtpAuth, os.Getenv("SMTP_FROM_ADDRESS"), []string{to}, concat(header, body)) + if err != nil { + logger.Errorf("cannot send data export error email: %v", err) + return errorCannotSendEmail + } + + return nil +} diff --git a/api/smtp_templates.go b/api/smtp_templates.go index ddf0d0a..c7f8e71 100644 --- a/api/smtp_templates.go +++ b/api/smtp_templates.go @@ -31,7 +31,7 @@ Subject: {{.Subject}} return errorMalformedTemplate } - names := []string{"confirm-hex", "reset-hex"} + names := []string{"confirm-hex", "reset-hex", "domain-export", "domain-export-error"} templates = make(map[string]*template.Template) diff --git a/db/20190131002240-export.sql b/db/20190131002240-export.sql new file mode 100644 index 0000000..d3df6bb --- /dev/null +++ b/db/20190131002240-export.sql @@ -0,0 +1,8 @@ +-- add export feature + +CREATE TABLE IF NOT EXISTS exports ( + exportHex TEXT NOT NULL UNIQUE PRIMARY KEY, + binData BYTEA NOT NULL, + domain TEXT NOT NULL, + creationDate TIMESTAMP NOT NULL +); diff --git a/frontend/dashboard.html b/frontend/dashboard.html index 61829a1..25b559b 100644 --- a/frontend/dashboard.html +++ b/frontend/dashboard.html @@ -82,27 +82,27 @@