From 86d059bf9970bd136b71a246babd4f907790b70f Mon Sep 17 00:00:00 2001 From: Dmitry Verkhoturov Date: Wed, 27 Jul 2022 02:52:13 +0200 Subject: [PATCH] move templates from rakyll/statik to go:embed There is no need for the rakyll/statik package starting with Go 1.16, which provides us with tools for embedding files without third-party libraries. --- Dockerfile | 1 - Dockerfile.artifacts | 2 - backend/app/cmd/server.go | 25 +----------- backend/app/cmd/server_test.go | 13 ------ backend/app/notify/email.go | 5 +-- backend/app/notify/email_test.go | 20 ++-------- backend/app/rest/api/rest.go | 2 - backend/app/rest/api/rest_private.go | 25 +++++------- backend/app/rest/api/rest_private_test.go | 7 ---- backend/app/rest/httperrors.go | 6 +-- backend/app/rest/httperrors_test.go | 9 +---- backend/app/store/formatter.go | 2 +- .../email_confirmation_login.html.tmpl | 0 .../email_confirmation_subscription.html.tmpl | 0 .../templates/static}/email_reply.html.tmpl | 0 .../static}/email_unsubscribe.html.tmpl | 0 .../static}/error_response.html.tmpl | 0 backend/app/templates/templates.go | 40 ++++++------------- backend/app/templates/templates_test.go | 13 ++---- site/src/docs/configuration/email/index.md | 2 +- site/src/docs/contributing/backend/index.md | 4 +- 21 files changed, 39 insertions(+), 137 deletions(-) rename backend/{templates => app/templates/static}/email_confirmation_login.html.tmpl (100%) rename backend/{templates => app/templates/static}/email_confirmation_subscription.html.tmpl (100%) rename backend/{templates => app/templates/static}/email_reply.html.tmpl (100%) rename backend/{templates => app/templates/static}/email_unsubscribe.html.tmpl (100%) rename backend/{templates => app/templates/static}/error_response.html.tmpl (100%) diff --git a/Dockerfile b/Dockerfile index 31054456d..420c2cf24 100644 --- a/Dockerfile +++ b/Dockerfile @@ -100,7 +100,6 @@ ADD backend/scripts/import.sh /usr/local/bin/import RUN chmod +x /entrypoint.sh /usr/local/bin/backup /usr/local/bin/restore /usr/local/bin/import COPY --from=build-backend /build/backend/remark42 /srv/remark42 -COPY --from=build-backend /build/backend/templates /srv COPY --from=build-frontend /srv/frontend/apps/remark42/public/ /srv/web/ COPY docker-init.sh /srv/init.sh RUN chown -R app:app /srv diff --git a/Dockerfile.artifacts b/Dockerfile.artifacts index 7a9d85d3b..53ff84235 100644 --- a/Dockerfile.artifacts +++ b/Dockerfile.artifacts @@ -36,8 +36,6 @@ RUN \ export WEB_ROOT=/build/backend/web && \ find . -regex '.*\.\(html\|js\|mjs\)$' -print -exec sed -i "s|{% REMARK_URL %}|http://127.0.0.1:8080|g" {} \; && \ statik --src=${WEB_ROOT} --dest=/build/backend/app/rest -p api -f && \ - statik --src=/build/backend/templates --dest=/build/backend/app -p templates -ns templates -f && \ - ls -la /build/backend/app/templates/statik.go && \ ls -la /build/backend/app/rest/api/statik.go && \ ls -la /build/backend/web/ diff --git a/backend/app/cmd/server.go b/backend/app/cmd/server.go index 8ed40b20e..9d8345579 100644 --- a/backend/app/cmd/server.go +++ b/backend/app/cmd/server.go @@ -877,11 +877,11 @@ func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) error { ContentType: s.Auth.Email.ContentType, } sndr := sender.NewEmailClient(params, log.Default()) - tmpl, err := s.loadEmailTemplate() + tmpl, err := templates.Read(s.Auth.Email.MsgTemplate) if err != nil { return err } - authenticator.AddVerifProvider("email", tmpl, sndr) + authenticator.AddVerifProvider("email", string(tmpl), sndr) } if s.Auth.Anonymous { @@ -925,27 +925,6 @@ func (s *ServerCommand) addAuthProviders(authenticator *auth.Service) error { return nil } -// loadEmailTemplate trying to get template from statik -func (s *ServerCommand) loadEmailTemplate() (string, error) { - var file []byte - var err error - - if s.Auth.Email.MsgTemplate == "email_confirmation_login.html.tmpl" { - fs := templates.NewFS() - file, err = fs.ReadFile(s.Auth.Email.MsgTemplate) - } else { - // deprecated loading from an external file, should be removed before v1.9.0 - file, err = os.ReadFile(s.Auth.Email.MsgTemplate) - log.Printf("[INFO] template %s will be read from disk", s.Auth.Email.MsgTemplate) - } - - if err != nil { - return "", fmt.Errorf("failed to read file %s: %w", s.Auth.Email.MsgTemplate, err) - } - - return string(file), nil -} - // creates and registers telegram auth, which we need separately from other auth providers func (s *ServerCommand) makeTelegramAuth(authenticator *auth.Service) providers.TGUpdatesReceiver { if s.Auth.Telegram { diff --git a/backend/app/cmd/server_test.go b/backend/app/cmd/server_test.go index 7f6c3518b..8b0c6281d 100644 --- a/backend/app/cmd/server_test.go +++ b/backend/app/cmd/server_test.go @@ -667,19 +667,6 @@ func TestServerAuthHooks(t *testing.T) { client.CloseIdleConnections() } -func TestServer_loadEmailTemplate(t *testing.T) { - cmd := ServerCommand{} - cmd.Auth.Email.MsgTemplate = "testdata/email.tmpl" - r, err := cmd.loadEmailTemplate() - assert.NoError(t, err) - assert.Equal(t, "The token is {{.Token}}", r) - - cmd.Auth.Email.MsgTemplate = "badpath.tmpl" - r, err = cmd.loadEmailTemplate() - assert.EqualError(t, err, "failed to read file badpath.tmpl: open badpath.tmpl: no such file or directory") - assert.Equal(t, r, "") -} - func TestServerCommand_parseSameSite(t *testing.T) { tbl := []struct { inp string diff --git a/backend/app/notify/email.go b/backend/app/notify/email.go index 87a1ce43e..186c7e507 100644 --- a/backend/app/notify/email.go +++ b/backend/app/notify/email.go @@ -100,7 +100,6 @@ func NewEmail(emailParams EmailParams, smtpParams ntf.SMTPParams) (*Email, error func (e *Email) setTemplates() error { var err error var msgTmplFile, verifyTmplFile []byte - fs := templates.NewFS() if e.VerificationTemplatePath == "" { e.VerificationTemplatePath = defaultEmailVerificationTemplatePath @@ -110,10 +109,10 @@ func (e *Email) setTemplates() error { e.MsgTemplatePath = defaultEmailTemplatePath } - if msgTmplFile, err = fs.ReadFile(e.MsgTemplatePath); err != nil { + if msgTmplFile, err = templates.Read(e.MsgTemplatePath); err != nil { return fmt.Errorf("can't read message template: %w", err) } - if verifyTmplFile, err = fs.ReadFile(e.VerificationTemplatePath); err != nil { + if verifyTmplFile, err = templates.Read(e.VerificationTemplatePath); err != nil { return fmt.Errorf("can't read verification template: %w", err) } if e.msgTmpl, err = template.New("msgTmpl").Parse(string(msgTmplFile)); err != nil { diff --git a/backend/app/notify/email_test.go b/backend/app/notify/email_test.go index 659a529fc..622882d0c 100644 --- a/backend/app/notify/email_test.go +++ b/backend/app/notify/email_test.go @@ -55,32 +55,18 @@ func Test_initTemplatesErr(t *testing.T) { errText string emailParams EmailParams }{ - { - name: "with wrong (default, working in prod) path to reply template", - errText: "can't read message template: open email_reply.html.tmpl: no such file or directory", - emailParams: EmailParams{}, - }, - { - name: "with wrong (default, working in prod) path to verification template", - errText: "can't read verification template: open email_confirmation_subscription.html.tmpl: no such file or directory", - emailParams: EmailParams{ - MsgTemplatePath: "testdata/msg.html.tmpl", - }, - }, { name: "with wrong path to verification template", - errText: "can't read verification template: open notfound.tmpl: no such file or directory", + errText: "notfound.tmpl: file does not exist", emailParams: EmailParams{ VerificationTemplatePath: "notfound.tmpl", - MsgTemplatePath: "testdata/msg.html.tmpl", }, }, { name: "with wrong path to message template", - errText: "can't read message template: open notfound.tmpl: no such file or directory", + errText: "notfound.tmpl: file does not exist", emailParams: EmailParams{ - VerificationTemplatePath: "testdata/verification.html.tmpl", - MsgTemplatePath: "notfound.tmpl", + MsgTemplatePath: "notfound.tmpl", }, }, { diff --git a/backend/app/rest/api/rest.go b/backend/app/rest/api/rest.go index bb6514826..820e95ac2 100644 --- a/backend/app/rest/api/rest.go +++ b/backend/app/rest/api/rest.go @@ -31,7 +31,6 @@ import ( "github.com/umputun/remark42/backend/app/store" "github.com/umputun/remark42/backend/app/store/image" "github.com/umputun/remark42/backend/app/store/service" - "github.com/umputun/remark42/backend/app/templates" ) // Rest is a rest access server @@ -377,7 +376,6 @@ func (s *Rest) controllerGroups() (public, private, admin, rss) { telegramService: s.TelegramService, remarkURL: s.RemarkURL, anonVote: s.AnonVote, - templates: templates.NewFS(), } admGrp := admin{ diff --git a/backend/app/rest/api/rest_private.go b/backend/app/rest/api/rest_private.go index dc3ee2230..f7b486420 100644 --- a/backend/app/rest/api/rest_private.go +++ b/backend/app/rest/api/rest_private.go @@ -43,7 +43,6 @@ type private struct { telegramService telegramService remarkURL string anonVote bool - templates templates.FileReader } // telegramService is a subset of Telegram service used for setting up user telegram notifications @@ -473,29 +472,26 @@ func (s *private) setConfirmedEmailCtrl(w http.ResponseWriter, r *http.Request) func (s *private) emailUnsubscribeCtrl(w http.ResponseWriter, r *http.Request) { tkn := r.URL.Query().Get("tkn") if tkn == "" { - rest.SendErrorHTML(w, r, http.StatusBadRequest, - fmt.Errorf("missing parameter"), "token parameter is required", rest.ErrInternal, s.templates) + rest.SendErrorHTML(w, r, http.StatusBadRequest, fmt.Errorf("missing parameter"), "token parameter is required", rest.ErrInternal) return } siteID := r.URL.Query().Get("site") confClaims, err := s.authenticator.TokenService().Parse(tkn) if err != nil { - rest.SendErrorHTML(w, r, http.StatusForbidden, err, "failed to verify confirmation token", rest.ErrInternal, s.templates) + rest.SendErrorHTML(w, r, http.StatusForbidden, err, "failed to verify confirmation token", rest.ErrInternal) return } if s.authenticator.TokenService().IsExpired(confClaims) { - rest.SendErrorHTML(w, r, http.StatusForbidden, - fmt.Errorf("expired"), "failed to verify confirmation token", rest.ErrInternal, s.templates) + rest.SendErrorHTML(w, r, http.StatusForbidden, fmt.Errorf("expired"), "failed to verify confirmation token", rest.ErrInternal) return } // Handshake.ID is user.ID + "::" + address elems := strings.Split(confClaims.Handshake.ID, "::") if len(elems) != 2 { - rest.SendErrorHTML(w, r, http.StatusBadRequest, - fmt.Errorf("%s", confClaims.Handshake.ID), "invalid handshake token", rest.ErrInternal, s.templates) + rest.SendErrorHTML(w, r, http.StatusBadRequest, fmt.Errorf("%s", confClaims.Handshake.ID), "invalid handshake token", rest.ErrInternal) return } userID := elems[0] @@ -508,14 +504,11 @@ func (s *private) emailUnsubscribeCtrl(w http.ResponseWriter, r *http.Request) { log.Printf("[WARN] can't read email for %s, %v", userID, err) } if existingAddress == "" { - rest.SendErrorHTML(w, r, http.StatusConflict, - fmt.Errorf("user is not subscribed"), "user does not have active email subscription", rest.ErrInternal, s.templates) + rest.SendErrorHTML(w, r, http.StatusConflict, fmt.Errorf("user is not subscribed"), "user does not have active email subscription", rest.ErrInternal) return } if address != existingAddress { - rest.SendErrorHTML(w, r, http.StatusBadRequest, - fmt.Errorf("wrong email unsubscription"), "email address in request does not match known for this user", - rest.ErrInternal, s.templates) + rest.SendErrorHTML(w, r, http.StatusBadRequest, fmt.Errorf("wrong email unsubscription"), "email address in request does not match known for this user", rest.ErrInternal) return } @@ -523,7 +516,7 @@ func (s *private) emailUnsubscribeCtrl(w http.ResponseWriter, r *http.Request) { if err = s.dataService.DeleteUserDetail(siteID, userID, engine.UserEmail); err != nil { code := parseError(err, rest.ErrInternal) - rest.SendErrorHTML(w, r, http.StatusBadRequest, err, "can't delete email for user", code, s.templates) + rest.SendErrorHTML(w, r, http.StatusBadRequest, err, "can't delete email for user", code) return } // clean User.Email from the token, if user has the token @@ -534,7 +527,7 @@ func (s *private) emailUnsubscribeCtrl(w http.ResponseWriter, r *http.Request) { if claims.User != nil && claims.User.Email != "" { claims.User.Email = "" if _, err = s.authenticator.TokenService().Set(w, claims); err != nil { - rest.SendErrorHTML(w, r, http.StatusInternalServerError, err, "failed to set token", rest.ErrInternal, s.templates) + rest.SendErrorHTML(w, r, http.StatusInternalServerError, err, "failed to set token", rest.ErrInternal) return } } @@ -546,7 +539,7 @@ func (s *private) emailUnsubscribeCtrl(w http.ResponseWriter, r *http.Request) { } } MustRead := func(path string) string { - file, err := s.templates.ReadFile(path) + file, err := templates.Read(path) if err != nil { panic(err) } diff --git a/backend/app/rest/api/rest_private_test.go b/backend/app/rest/api/rest_private_test.go index d3a6eecd8..203f45512 100644 --- a/backend/app/rest/api/rest_private_test.go +++ b/backend/app/rest/api/rest_private_test.go @@ -600,17 +600,10 @@ func TestRest_AnonVote(t *testing.T) { assert.Equal(t, map[string]store.VotedIPInfo(nil), cr.VotedIPs) } -type MockFS struct{} - -func (fs *MockFS) ReadFile(path string) ([]byte, error) { - return []byte(fmt.Sprintf("template %s", path)), nil -} - func TestRest_EmailAndTelegram(t *testing.T) { ts, srv, teardown := startupT(t) defer teardown() - srv.privRest.templates = &MockFS{} srv.privRest.telegramService = &mockTelegram{site: "remark42"} // issue good token diff --git a/backend/app/rest/httperrors.go b/backend/app/rest/httperrors.go index e8468ab19..93c56e1c2 100644 --- a/backend/app/rest/httperrors.go +++ b/backend/app/rest/httperrors.go @@ -48,9 +48,9 @@ type errTmplData struct { Details string } -// SendErrorHTML makes html body with provided template and responds with provided http status code, +// SendErrorHTML makes html body from error_response.html.tmpl template and responds with provided http status code, // error code is not included in render as it is intended for UI developers and not for the users -func SendErrorHTML(w http.ResponseWriter, r *http.Request, httpStatusCode int, err error, details string, errCode int, t templates.FileReader) { +func SendErrorHTML(w http.ResponseWriter, r *http.Request, httpStatusCode int, err error, details string, errCode int) { // MustExecute behaves like template.Execute, but panics if an error occurs. MustExecute := func(tmpl *template.Template, wr io.Writer, data interface{}) { if err = tmpl.Execute(wr, data); err != nil { @@ -58,7 +58,7 @@ func SendErrorHTML(w http.ResponseWriter, r *http.Request, httpStatusCode int, e } } MustRead := func(path string) string { - file, e := t.ReadFile(path) + file, e := templates.Read(path) if e != nil { panic(e) } diff --git a/backend/app/rest/httperrors_test.go b/backend/app/rest/httperrors_test.go index 5b4e73088..cc95b992a 100644 --- a/backend/app/rest/httperrors_test.go +++ b/backend/app/rest/httperrors_test.go @@ -36,18 +36,11 @@ func TestSendErrorJSON(t *testing.T) { assert.Equal(t, `{"code":123,"details":"error details 123456","error":"error 500"}`+"\n", string(body)) } -type MockFS struct{} - -func (fs *MockFS) ReadFile(path string) ([]byte, error) { - return []byte(fmt.Sprintf("{{.Error}}{{.Details}} %s", path)), nil -} - func TestSendErrorHTML(t *testing.T) { - fs := &MockFS{} ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/error" { t.Log("http err request", r.URL) - SendErrorHTML(w, r, 500, fmt.Errorf("error 500"), "error details 123456", 987, fs) + SendErrorHTML(w, r, 500, fmt.Errorf("error 500"), "error details 123456", 987) return } w.WriteHeader(404) diff --git a/backend/app/store/formatter.go b/backend/app/store/formatter.go index 3eb3f638c..26189c85f 100644 --- a/backend/app/store/formatter.go +++ b/backend/app/store/formatter.go @@ -4,7 +4,7 @@ import ( "net/url" "strings" - bfchroma "github.com/Depado/bfchroma/v2" + "github.com/Depado/bfchroma/v2" "github.com/PuerkitoBio/goquery" "github.com/alecthomas/chroma/v2/formatters/html" bf "github.com/russross/blackfriday/v2" diff --git a/backend/templates/email_confirmation_login.html.tmpl b/backend/app/templates/static/email_confirmation_login.html.tmpl similarity index 100% rename from backend/templates/email_confirmation_login.html.tmpl rename to backend/app/templates/static/email_confirmation_login.html.tmpl diff --git a/backend/templates/email_confirmation_subscription.html.tmpl b/backend/app/templates/static/email_confirmation_subscription.html.tmpl similarity index 100% rename from backend/templates/email_confirmation_subscription.html.tmpl rename to backend/app/templates/static/email_confirmation_subscription.html.tmpl diff --git a/backend/templates/email_reply.html.tmpl b/backend/app/templates/static/email_reply.html.tmpl similarity index 100% rename from backend/templates/email_reply.html.tmpl rename to backend/app/templates/static/email_reply.html.tmpl diff --git a/backend/templates/email_unsubscribe.html.tmpl b/backend/app/templates/static/email_unsubscribe.html.tmpl similarity index 100% rename from backend/templates/email_unsubscribe.html.tmpl rename to backend/app/templates/static/email_unsubscribe.html.tmpl diff --git a/backend/templates/error_response.html.tmpl b/backend/app/templates/static/error_response.html.tmpl similarity index 100% rename from backend/templates/error_response.html.tmpl rename to backend/app/templates/static/error_response.html.tmpl diff --git a/backend/app/templates/templates.go b/backend/app/templates/templates.go index 712b25e29..969c1788d 100644 --- a/backend/app/templates/templates.go +++ b/backend/app/templates/templates.go @@ -1,38 +1,24 @@ package templates import ( - "net/http" + "embed" + "io/fs" "os" "path/filepath" - - log "github.com/go-pkgz/lgr" - "github.com/rakyll/statik/fs" ) -// FS stores link to statikFS if it exists -type FS struct { - statik http.FileSystem -} - -// FileReader describes methods of filesystem -type FileReader interface { - ReadFile(path string) ([]byte, error) -} +//go:embed static +var templateFS embed.FS -// NewFS returns new FS instance, which will read from statik if it's available and from fs otherwise -func NewFS() *FS { - f := &FS{} - if statikFS, err := fs.NewWithNamespace("templates"); err == nil { - log.Printf("[INFO] templates will be read from statik") - f.statik = statikFS +// Read reads either template from disk if it exists, or from embedded template +func Read(path string) ([]byte, error) { + if _, err := os.Stat(filepath.Clean(path)); err == nil { + return os.ReadFile(filepath.Clean(path)) } - return f -} - -// ReadFile depends on statik achieve exists -func (f *FS) ReadFile(path string) ([]byte, error) { - if f.statik != nil { - return fs.ReadFile(f.statik, filepath.Join("/", path)) //nolint:gocritic // root folder is a requirement for statik + // remove "static/" prefix from path + var contentFS, err = fs.Sub(templateFS, "static") + if err != nil { + return nil, err } - return os.ReadFile(filepath.Clean(path)) + return fs.ReadFile(contentFS, filepath.Clean(path)) } diff --git a/backend/app/templates/templates_test.go b/backend/app/templates/templates_test.go index 3ccd968e0..80624048a 100644 --- a/backend/app/templates/templates_test.go +++ b/backend/app/templates/templates_test.go @@ -6,19 +6,12 @@ import ( "github.com/stretchr/testify/assert" ) -func TestNewFS(t *testing.T) { - fs := NewFS() - assert.NotNil(t, &fs) -} - -func TestFS_ReadFile(t *testing.T) { - fs := NewFS() - - file, err := fs.ReadFile("testdata/template.html.tmpl") +func Test_Read(t *testing.T) { + file, err := Read("testdata/template.html.tmpl") assert.NoError(t, err) assert.Equal(t, []byte("template\n"), file) - file, err = fs.ReadFile("testdata/bad_path.html.tmpl") + file, err = Read("testdata/bad_path.html.tmpl") assert.Error(t, err) assert.Nil(t, file) } diff --git a/site/src/docs/configuration/email/index.md b/site/src/docs/configuration/email/index.md index 9831d7f1b..51e956067 100644 --- a/site/src/docs/configuration/email/index.md +++ b/site/src/docs/configuration/email/index.md @@ -190,7 +190,7 @@ After you set `SMTP_` variables, you can allow email authentication by setting t ## HTML templates for emails and error messages -Remark42 uses golang templates for email templating. Templates are located in `backend/templates` and embedded into binary by statik +Remark42 uses golang templates for email templating. Templates are located in `backend/app/templates/static` and embedded into binary by `go:embed` [directive](https://pkg.go.dev/embed). Now we have the following templates: diff --git a/site/src/docs/contributing/backend/index.md b/site/src/docs/contributing/backend/index.md index 420b0db7c..e725d3982 100644 --- a/site/src/docs/contributing/backend/index.md +++ b/site/src/docs/contributing/backend/index.md @@ -30,7 +30,7 @@ Run tests in your IDE, and re-run `make rundev` each time you want to see how yo You have to [install](https://golang.org/doc/install) the latest stable `go` toolchain to run the backend locally. -In order to have working Remark42 installation you need once to copy frontend static files to `./backend/web` directory from `master` docker image, and also copy files from `./templates` to the `./backend` as they are expected to be where application starts: +In order to have working Remark42 installation you need once to copy frontend static files to `./backend/web` directory from `master` docker image, as it is expected to be where application compiles: ```shell # frontend files @@ -38,8 +38,6 @@ docker pull umputun/remark42:master docker create -ti --name remark42files umputun/remark42:master sh docker cp remark42files:/srv/web/ ./backend/ docker rm -f remark42files -# template files -cp ./backend/templates/* ./backend # fix frontend files to point to the right URL ## Mac version find -E ./backend/web -regex '.*\.(html|js|mjs)$' -print -exec sed -i '' "s|{% REMARK_URL %}|http://127.0.0.1:8080|g" {} \;