From 565c30eac9e00b2ebcbdbb8e05b5e8238a15fefb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B8rn=20Erik=20Pedersen?= Date: Mon, 16 Dec 2024 08:34:17 +0100 Subject: [PATCH] js: Fix js.Batch for multihost setups Note that this is an unreleased feature. Fixes #13151 --- deps/deps.go | 7 ++ hugolib/paths/paths.go | 2 +- hugolib/site.go | 8 +- internal/js/api.go | 51 +++++++++ internal/js/esbuild/batch.go | 105 ++++++++++-------- internal/js/esbuild/batch_integration_test.go | 63 +++++++++++ resources/resource.go | 7 -- tpl/js/js.go | 18 +-- 8 files changed, 190 insertions(+), 71 deletions(-) create mode 100644 internal/js/api.go diff --git a/deps/deps.go b/deps/deps.go index 8e9ec42d8b6..56a3d36446a 100644 --- a/deps/deps.go +++ b/deps/deps.go @@ -24,6 +24,7 @@ import ( "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/hugofs" "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/internal/js" "github.com/gohugoio/hugo/internal/warpc" "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/resources/page" @@ -105,6 +106,12 @@ type Deps struct { // TODO(bep) rethink this re. a plugin setup, but this will have to do for now. WasmDispatchers *warpc.Dispatchers + // The JS batcher client. + JSBatcherClient js.BatcherClient + + // The JS batcher client. + // JSBatcherClient *esbuild.BatcherClient + isClosed bool *globalErrHandler diff --git a/hugolib/paths/paths.go b/hugolib/paths/paths.go index 397dba3f809..60ec873f97b 100644 --- a/hugolib/paths/paths.go +++ b/hugolib/paths/paths.go @@ -67,7 +67,7 @@ func New(fs *hugofs.Fs, cfg config.AllProvider) (*Paths, error) { var multihostTargetBasePaths []string if cfg.IsMultihost() && len(cfg.Languages()) > 1 { for _, l := range cfg.Languages() { - multihostTargetBasePaths = append(multihostTargetBasePaths, l.Lang) + multihostTargetBasePaths = append(multihostTargetBasePaths, hpaths.ToSlashPreserveLeading(l.Lang)) } } diff --git a/hugolib/site.go b/hugolib/site.go index f73bd2517e1..4e2497ee1f1 100644 --- a/hugolib/site.go +++ b/hugolib/site.go @@ -42,6 +42,7 @@ import ( "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/hugolib/doctree" "github.com/gohugoio/hugo/hugolib/pagesfromdata" + "github.com/gohugoio/hugo/internal/js/esbuild" "github.com/gohugoio/hugo/internal/warpc" "github.com/gohugoio/hugo/langs/i18n" "github.com/gohugoio/hugo/modules" @@ -205,6 +206,12 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { return nil, err } + batcherClient, err := esbuild.NewBatcherClient(firstSiteDeps) + if err != nil { + return nil, err + } + firstSiteDeps.JSBatcherClient = batcherClient + confm := cfg.Configs if err := confm.Validate(logger); err != nil { return nil, err @@ -313,7 +320,6 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) { return li.Lang < lj.Lang }) - var err error h, err = newHugoSites(cfg, firstSiteDeps, pageTrees, sites) if err == nil && h == nil { panic("hugo: newHugoSitesNew returned nil error and nil HugoSites") diff --git a/internal/js/api.go b/internal/js/api.go new file mode 100644 index 00000000000..30180dece07 --- /dev/null +++ b/internal/js/api.go @@ -0,0 +1,51 @@ +// Copyright 2024 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package js + +import ( + "context" + + "github.com/gohugoio/hugo/common/maps" + "github.com/gohugoio/hugo/resources/resource" +) + +// BatcherClient is used to do JS batch operations. +type BatcherClient interface { + New(id string) (Batcher, error) + Store() *maps.Cache[string, Batcher] +} + +// BatchPackage holds a group of JavaScript resources. +type BatchPackage interface { + Groups() map[string]resource.Resources +} + +// Batcher is used to build JavaScript packages. +type Batcher interface { + Build(context.Context) (BatchPackage, error) + Config(ctx context.Context) OptionsSetter + Group(ctx context.Context, id string) BatcherGroup +} + +// BatcherGroup is a group of scripts and instances. +type BatcherGroup interface { + Instance(sid, iid string) OptionsSetter + Runner(id string) OptionsSetter + Script(id string) OptionsSetter +} + +// OptionsSetter is used to set options for a batch, script or instance. +type OptionsSetter interface { + SetOptions(map[string]any) string +} diff --git a/internal/js/esbuild/batch.go b/internal/js/esbuild/batch.go index d0b6dba3337..c5394ac0a9c 100644 --- a/internal/js/esbuild/batch.go +++ b/internal/js/esbuild/batch.go @@ -20,6 +20,7 @@ import ( _ "embed" "encoding/json" "fmt" + "io" "path" "path/filepath" "reflect" @@ -34,7 +35,9 @@ import ( "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/common/paths" "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/helpers" "github.com/gohugoio/hugo/identity" + "github.com/gohugoio/hugo/internal/js" "github.com/gohugoio/hugo/lazy" "github.com/gohugoio/hugo/media" "github.com/gohugoio/hugo/resources" @@ -42,11 +45,10 @@ import ( "github.com/gohugoio/hugo/resources/resource_factories/create" "github.com/gohugoio/hugo/tpl" "github.com/mitchellh/mapstructure" - "github.com/spf13/afero" "github.com/spf13/cast" ) -var _ Batcher = (*batcher)(nil) +var _ js.Batcher = (*batcher)(nil) const ( NsBatch = "_hugo-js-batch" @@ -58,7 +60,7 @@ const ( //go:embed batch-esm-runner.gotmpl var runnerTemplateStr string -var _ BatchPackage = (*Package)(nil) +var _ js.BatchPackage = (*Package)(nil) var _ buildToucher = (*optsHolder[scriptOptions])(nil) @@ -67,16 +69,17 @@ var ( _ isBuiltOrTouchedProvider = (*scriptGroup)(nil) ) -func NewBatcherClient(deps *deps.Deps) (*BatcherClient, error) { +func NewBatcherClient(deps *deps.Deps) (js.BatcherClient, error) { c := &BatcherClient{ d: deps, buildClient: NewBuildClient(deps.BaseFs.Assets, deps.ResourceSpec), createClient: create.New(deps.ResourceSpec), - bundlesCache: maps.NewCache[string, BatchPackage](), + batcherStore: maps.NewCache[string, js.Batcher](), + bundlesStore: maps.NewCache[string, js.BatchPackage](), } deps.BuildEndListeners.Add(func(...any) bool { - c.bundlesCache.Reset() + c.bundlesStore.Reset() return false }) @@ -125,7 +128,7 @@ func (o *opts[K, C]) Reset() { o.h.resetCounter++ } -func (o *opts[K, C]) Get(id uint32) OptionsSetter { +func (o *opts[K, C]) Get(id uint32) js.OptionsSetter { var b *optsHolder[C] o.once.Do(func() { b = o.h @@ -184,18 +187,6 @@ func newOpts[K any, C optionsCompiler[C]](key K, optionsID string, defaults defa } } -// BatchPackage holds a group of JavaScript resources. -type BatchPackage interface { - Groups() map[string]resource.Resources -} - -// Batcher is used to build JavaScript packages. -type Batcher interface { - Build(context.Context) (BatchPackage, error) - Config(ctx context.Context) OptionsSetter - Group(ctx context.Context, id string) BatcherGroup -} - // BatcherClient is a client for building JavaScript packages. type BatcherClient struct { d *deps.Deps @@ -206,12 +197,13 @@ type BatcherClient struct { createClient *create.Client buildClient *BuildClient - bundlesCache *maps.Cache[string, BatchPackage] + batcherStore *maps.Cache[string, js.Batcher] + bundlesStore *maps.Cache[string, js.BatchPackage] } // New creates a new Batcher with the given ID. // This will be typically created once and reused across rebuilds. -func (c *BatcherClient) New(id string) (Batcher, error) { +func (c *BatcherClient) New(id string) (js.Batcher, error) { var initErr error c.once.Do(func() { // We should fix the initialization order here (or use the Go template package directly), but we need to wait @@ -288,6 +280,10 @@ func (c *BatcherClient) New(id string) (Batcher, error) { return b, nil } +func (c *BatcherClient) Store() *maps.Cache[string, js.Batcher] { + return c.batcherStore +} + func (c *BatcherClient) buildBatchGroup(ctx context.Context, t *batchGroupTemplateContext) (resource.Resource, string, error) { var buf bytes.Buffer @@ -304,18 +300,6 @@ func (c *BatcherClient) buildBatchGroup(ctx context.Context, t *batchGroupTempla return r, s, nil } -// BatcherGroup is a group of scripts and instances. -type BatcherGroup interface { - Instance(sid, iid string) OptionsSetter - Runner(id string) OptionsSetter - Script(id string) OptionsSetter -} - -// OptionsSetter is used to set options for a batch, script or instance. -type OptionsSetter interface { - SetOptions(map[string]any) string -} - // Package holds a group of JavaScript resources. type Package struct { id string @@ -353,9 +337,9 @@ type batcher struct { } // Build builds the batch if not already built or if it's stale. -func (b *batcher) Build(ctx context.Context) (BatchPackage, error) { +func (b *batcher) Build(ctx context.Context) (js.BatchPackage, error) { key := dynacache.CleanKey(b.id + ".js") - p, err := b.client.bundlesCache.GetOrCreate(key, func() (BatchPackage, error) { + p, err := b.client.bundlesStore.GetOrCreate(key, func() (js.BatchPackage, error) { return b.build(ctx) }) if err != nil { @@ -364,11 +348,11 @@ func (b *batcher) Build(ctx context.Context) (BatchPackage, error) { return p, nil } -func (b *batcher) Config(ctx context.Context) OptionsSetter { +func (b *batcher) Config(ctx context.Context) js.OptionsSetter { return b.configOptions.Get(b.buildCount) } -func (b *batcher) Group(ctx context.Context, id string) BatcherGroup { +func (b *batcher) Group(ctx context.Context, id string) js.BatcherGroup { if err := ValidateBatchID(id, false); err != nil { panic(err) } @@ -419,7 +403,7 @@ func (b *batcher) isStale() bool { return false } -func (b *batcher) build(ctx context.Context) (BatchPackage, error) { +func (b *batcher) build(ctx context.Context) (js.BatchPackage, error) { b.mu.Lock() defer b.mu.Unlock() defer func() { @@ -463,6 +447,8 @@ func (b *batcher) doBuild(ctx context.Context) (*Package, error) { pathGroup: maps.NewCache[string, string](), } + multihostBasePaths := b.client.d.ResourceSpec.MultihostTargetBasePaths + // Entry points passed to ESBuid. var entryPoints []string addResource := func(group, pth string, r resource.Resource, isResult bool) { @@ -701,15 +687,36 @@ func (b *batcher) doBuild(ctx context.Context) (*Package, error) { if !handled { // Copy to destination. - p := strings.TrimPrefix(o.Path, outDir) - targetFilename := filepath.Join(b.id, p) - fs := b.client.d.BaseFs.PublishFs - if err := fs.MkdirAll(filepath.Dir(targetFilename), 0o777); err != nil { - return nil, fmt.Errorf("failed to create dir %q: %w", targetFilename, err) + // In a multihost setup, we will have multiple targets. + var targetFilenames []string + if len(multihostBasePaths) > 0 { + for _, base := range multihostBasePaths { + p := strings.TrimPrefix(o.Path, outDir) + targetFilename := filepath.Join(base, b.id, p) + targetFilenames = append(targetFilenames, targetFilename) + } + } else { + p := strings.TrimPrefix(o.Path, outDir) + targetFilename := filepath.Join(b.id, p) + targetFilenames = append(targetFilenames, targetFilename) } - if err := afero.WriteFile(fs, targetFilename, o.Contents, 0o666); err != nil { - return nil, fmt.Errorf("failed to write to %q: %w", targetFilename, err) + fs := b.client.d.BaseFs.PublishFs + + if err := func() error { + fw, err := helpers.OpenFilesForWriting(fs, targetFilenames...) + if err != nil { + return err + } + defer fw.Close() + + fr := bytes.NewReader(o.Contents) + + _, err = io.Copy(fw, fr) + + return err + }(); err != nil { + return nil, fmt.Errorf("failed to copy to %q: %w", targetFilenames, err) } } } @@ -845,7 +852,7 @@ type optionsGetSetter[K, C any] interface { Key() K Reset() - Get(uint32) OptionsSetter + Get(uint32) js.OptionsSetter isStale() bool currPrev() (map[string]any, map[string]any) } @@ -975,7 +982,7 @@ func (b *scriptGroup) IdentifierBase() string { return b.id } -func (s *scriptGroup) Instance(sid, id string) OptionsSetter { +func (s *scriptGroup) Instance(sid, id string) js.OptionsSetter { if err := ValidateBatchID(sid, false); err != nil { panic(err) } @@ -1014,7 +1021,7 @@ func (g *scriptGroup) Reset() { } } -func (s *scriptGroup) Runner(id string) OptionsSetter { +func (s *scriptGroup) Runner(id string) js.OptionsSetter { if err := ValidateBatchID(id, false); err != nil { panic(err) } @@ -1043,7 +1050,7 @@ func (s *scriptGroup) Runner(id string) OptionsSetter { return s.runnersOptions[sid].Get(s.b.buildCount) } -func (s *scriptGroup) Script(id string) OptionsSetter { +func (s *scriptGroup) Script(id string) js.OptionsSetter { if err := ValidateBatchID(id, false); err != nil { panic(err) } diff --git a/internal/js/esbuild/batch_integration_test.go b/internal/js/esbuild/batch_integration_test.go index 3501f820a86..55528bdf042 100644 --- a/internal/js/esbuild/batch_integration_test.go +++ b/internal/js/esbuild/batch_integration_test.go @@ -184,6 +184,69 @@ func TestBatchEditScriptParam(t *testing.T) { b.AssertFileContent("public/mybatch/mygroup.js", "param-p1-main-edited") } +func TestBatchMultiHost(t *testing.T) { + files := ` +-- hugo.toml -- +disableKinds = ["taxonomy", "term", "section"] +[languages] +[languages.en] +weight = 1 +baseURL = "https://example.com/en" +[languages.fr] +weight = 2 +baseURL = "https://example.com/fr" +disableLiveReload = true +-- assets/js/styles.css -- +body { + background-color: red; +} +-- assets/js/main.js -- +import * as foo from 'mylib'; +console.log("Hello, Main!"); +-- assets/js/runner.js -- +console.log("Hello, Runner!"); +-- node_modules/mylib/index.js -- +console.log("Hello, My Lib!"); +-- layouts/index.html -- +Home. +{{ $batch := (js.Batch "mybatch") }} + {{ with $batch.Config }} + {{ .SetOptions (dict + "params" (dict "id" "config") + "sourceMap" "" + ) + }} +{{ end }} +{{ with (templates.Defer (dict "key" "global")) }} +Defer: +{{ $batch := (js.Batch "mybatch") }} +{{ range $k, $v := $batch.Build.Groups }} + {{ range $kk, $vv := . -}} + {{ $k }}: {{ .RelPermalink }} + {{ end }} +{{ end -}} +{{ end }} +{{ $batch := (js.Batch "mybatch") }} +{{ with $batch.Group "mygroup" }} + {{ with .Runner "run" }} + {{ .SetOptions (dict "resource" (resources.Get "js/runner.js")) }} + {{ end }} + {{ with .Script "main" }} + {{ .SetOptions (dict "resource" (resources.Get "js/main.js") "params" (dict "p1" "param-p1-main" )) }} + {{ end }} + {{ with .Instance "main" "i1" }} + {{ .SetOptions (dict "params" (dict "title" "Instance 1")) }} + {{ end }} +{{ end }} + + +` + b := hugolib.Test(t, files, hugolib.TestOptWithOSFs()) + b.AssertPublishDir( + "en/mybatch/chunk-TOZKWCDE.js", "en/mybatch/mygroup.js ", + "fr/mybatch/mygroup.js", "fr/mybatch/chunk-TOZKWCDE.js") +} + func TestBatchRenameBundledScript(t *testing.T) { files := jsBatchFilesTemplate b := hugolib.TestRunning(t, files, hugolib.TestOptWithOSFs()) diff --git a/resources/resource.go b/resources/resource.go index 6025cbf4c3e..4b81a478a42 100644 --- a/resources/resource.go +++ b/resources/resource.go @@ -141,13 +141,6 @@ func (fd *ResourceSourceDescriptor) init(r *Spec) error { } fd.TargetPath = paths.ToSlashPreserveLeading(fd.TargetPath) - for i, base := range fd.TargetBasePaths { - dir := paths.ToSlashPreserveLeading(base) - if dir == "/" { - dir = "" - } - fd.TargetBasePaths[i] = dir - } if fd.NameNormalized == "" { fd.NameNormalized = fd.TargetPath diff --git a/tpl/js/js.go b/tpl/js/js.go index b686b76a7bc..dfd0a358118 100644 --- a/tpl/js/js.go +++ b/tpl/js/js.go @@ -17,8 +17,8 @@ package js import ( "errors" - "github.com/gohugoio/hugo/common/maps" "github.com/gohugoio/hugo/deps" + "github.com/gohugoio/hugo/internal/js" "github.com/gohugoio/hugo/internal/js/esbuild" "github.com/gohugoio/hugo/resources" "github.com/gohugoio/hugo/resources/resource" @@ -34,16 +34,9 @@ func New(d *deps.Deps) (*Namespace, error) { return &Namespace{}, nil } - batcherClient, err := esbuild.NewBatcherClient(d) - if err != nil { - return nil, err - } - return &Namespace{ d: d, jsTransformClient: jstransform.New(d.BaseFs.Assets, d.ResourceSpec), - jsBatcherClient: batcherClient, - jsBatcherStore: maps.NewCache[string, esbuild.Batcher](), createClient: create.New(d.ResourceSpec), babelClient: babel.New(d.ResourceSpec), }, nil @@ -56,8 +49,6 @@ type Namespace struct { jsTransformClient *jstransform.Client createClient *create.Client babelClient *babel.Client - jsBatcherClient *esbuild.BatcherClient - jsBatcherStore *maps.Cache[string, esbuild.Batcher] } // Build processes the given Resource with ESBuild. @@ -90,12 +81,13 @@ func (ns *Namespace) Build(args ...any) (resource.Resource, error) { // Repeated calls with the same ID will return the same Batcher. // The ID will be used to name the root directory of the batch. // Forward slashes in the ID is allowed. -func (ns *Namespace) Batch(id string) (esbuild.Batcher, error) { +func (ns *Namespace) Batch(id string) (js.Batcher, error) { if err := esbuild.ValidateBatchID(id, true); err != nil { return nil, err } - b, err := ns.jsBatcherStore.GetOrCreate(id, func() (esbuild.Batcher, error) { - return ns.jsBatcherClient.New(id) + + b, err := ns.d.JSBatcherClient.Store().GetOrCreate(id, func() (js.Batcher, error) { + return ns.d.JSBatcherClient.New(id) }) if err != nil { return nil, err