js: Fix js.Batch for multihost setups

Note that this is an unreleased feature. Fixes #13151
gohugoio · Dec 16, 2024 · 565c30e · 565c30e
1 parent 48dd6a9
commit 565c30e
Show file tree

Hide file tree

Showing 8 changed files with 190 additions and 71 deletions.
diff --git a/deps/deps.go b/deps/deps.go
@@ -24,6 +24,7 @@ import (
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugofs"
 	"github.com/gohugoio/hugo/identity"
+	"github.com/gohugoio/hugo/internal/js"
 	"github.com/gohugoio/hugo/internal/warpc"
 	"github.com/gohugoio/hugo/media"
 	"github.com/gohugoio/hugo/resources/page"
@@ -105,6 +106,12 @@ type Deps struct {
 	// TODO(bep) rethink this re. a plugin setup, but this will have to do for now.
 	WasmDispatchers *warpc.Dispatchers
 
+	// The JS batcher client.
+	JSBatcherClient js.BatcherClient
+
+	// The JS batcher client.
+	// JSBatcherClient *esbuild.BatcherClient
+
 	isClosed bool
 
 	*globalErrHandler

diff --git a/hugolib/paths/paths.go b/hugolib/paths/paths.go
@@ -67,7 +67,7 @@ func New(fs *hugofs.Fs, cfg config.AllProvider) (*Paths, error) {
 	var multihostTargetBasePaths []string
 	if cfg.IsMultihost() && len(cfg.Languages()) > 1 {
 		for _, l := range cfg.Languages() {
-			multihostTargetBasePaths = append(multihostTargetBasePaths, l.Lang)
+			multihostTargetBasePaths = append(multihostTargetBasePaths, hpaths.ToSlashPreserveLeading(l.Lang))
 		}
 	}
 

diff --git a/hugolib/site.go b/hugolib/site.go
@@ -42,6 +42,7 @@ import (
 	"github.com/gohugoio/hugo/deps"
 	"github.com/gohugoio/hugo/hugolib/doctree"
 	"github.com/gohugoio/hugo/hugolib/pagesfromdata"
+	"github.com/gohugoio/hugo/internal/js/esbuild"
 	"github.com/gohugoio/hugo/internal/warpc"
 	"github.com/gohugoio/hugo/langs/i18n"
 	"github.com/gohugoio/hugo/modules"
@@ -205,6 +206,12 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
 		return nil, err
 	}
 
+	batcherClient, err := esbuild.NewBatcherClient(firstSiteDeps)
+	if err != nil {
+		return nil, err
+	}
+	firstSiteDeps.JSBatcherClient = batcherClient
+
 	confm := cfg.Configs
 	if err := confm.Validate(logger); err != nil {
 		return nil, err
@@ -313,7 +320,6 @@ func NewHugoSites(cfg deps.DepsCfg) (*HugoSites, error) {
 		return li.Lang < lj.Lang
 	})
 
-	var err error
 	h, err = newHugoSites(cfg, firstSiteDeps, pageTrees, sites)
 	if err == nil && h == nil {
 		panic("hugo: newHugoSitesNew returned nil error and nil HugoSites")

diff --git a/internal/js/api.go b/internal/js/api.go
@@ -0,0 +1,51 @@
+// Copyright 2024 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package js
+
+import (
+	"context"
+
+	"github.com/gohugoio/hugo/common/maps"
+	"github.com/gohugoio/hugo/resources/resource"
+)
+
+// BatcherClient is used to do JS batch operations.
+type BatcherClient interface {
+	New(id string) (Batcher, error)
+	Store() *maps.Cache[string, Batcher]
+}
+
+// BatchPackage holds a group of JavaScript resources.
+type BatchPackage interface {
+	Groups() map[string]resource.Resources
+}
+
+// Batcher is used to build JavaScript packages.
+type Batcher interface {
+	Build(context.Context) (BatchPackage, error)
+	Config(ctx context.Context) OptionsSetter
+	Group(ctx context.Context, id string) BatcherGroup
+}
+
+// BatcherGroup is a group of scripts and instances.
+type BatcherGroup interface {
+	Instance(sid, iid string) OptionsSetter
+	Runner(id string) OptionsSetter
+	Script(id string) OptionsSetter
+}
+
+// OptionsSetter is used to set options for a batch, script or instance.
+type OptionsSetter interface {
+	SetOptions(map[string]any) string
+}
diff --git a/internal/js/esbuild/batch.go b/internal/js/esbuild/batch.go
@@ -20,6 +20,7 @@ import (
 	_ "embed"
 	"encoding/json"
 	"fmt"
+	"io"
 	"path"
 	"path/filepath"
 	"reflect"
@@ -34,19 +35,20 @@ import (
 	"github.com/gohugoio/hugo/common/maps"
 	"github.com/gohugoio/hugo/common/paths"
 	"github.com/gohugoio/hugo/deps"
+	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/identity"
+	"github.com/gohugoio/hugo/internal/js"
 	"github.com/gohugoio/hugo/lazy"
 	"github.com/gohugoio/hugo/media"
 	"github.com/gohugoio/hugo/resources"
 	"github.com/gohugoio/hugo/resources/resource"
 	"github.com/gohugoio/hugo/resources/resource_factories/create"
 	"github.com/gohugoio/hugo/tpl"
 	"github.com/mitchellh/mapstructure"
-	"github.com/spf13/afero"
 	"github.com/spf13/cast"
 )
 
-var _ Batcher = (*batcher)(nil)
+var _ js.Batcher = (*batcher)(nil)
 
 const (
 	NsBatch = "_hugo-js-batch"
@@ -58,7 +60,7 @@ const (
 //go:embed batch-esm-runner.gotmpl
 var runnerTemplateStr string
 
-var _ BatchPackage = (*Package)(nil)
+var _ js.BatchPackage = (*Package)(nil)
 
 var _ buildToucher = (*optsHolder[scriptOptions])(nil)
 
@@ -67,16 +69,17 @@ var (
 	_ isBuiltOrTouchedProvider = (*scriptGroup)(nil)
 )
 
-func NewBatcherClient(deps *deps.Deps) (*BatcherClient, error) {
+func NewBatcherClient(deps *deps.Deps) (js.BatcherClient, error) {
 	c := &BatcherClient{
 		d:            deps,
 		buildClient:  NewBuildClient(deps.BaseFs.Assets, deps.ResourceSpec),
 		createClient: create.New(deps.ResourceSpec),
-		bundlesCache: maps.NewCache[string, BatchPackage](),
+		batcherStore: maps.NewCache[string, js.Batcher](),
+		bundlesStore: maps.NewCache[string, js.BatchPackage](),
 	}
 
 	deps.BuildEndListeners.Add(func(...any) bool {
-		c.bundlesCache.Reset()
+		c.bundlesStore.Reset()
 		return false
 	})
 
@@ -125,7 +128,7 @@ func (o *opts[K, C]) Reset() {
 	o.h.resetCounter++
 }
 
-func (o *opts[K, C]) Get(id uint32) OptionsSetter {
+func (o *opts[K, C]) Get(id uint32) js.OptionsSetter {
 	var b *optsHolder[C]
 	o.once.Do(func() {
 		b = o.h
@@ -184,18 +187,6 @@ func newOpts[K any, C optionsCompiler[C]](key K, optionsID string, defaults defa
 	}
 }
 
-// BatchPackage holds a group of JavaScript resources.
-type BatchPackage interface {
-	Groups() map[string]resource.Resources
-}
-
-// Batcher is used to build JavaScript packages.
-type Batcher interface {
-	Build(context.Context) (BatchPackage, error)
-	Config(ctx context.Context) OptionsSetter
-	Group(ctx context.Context, id string) BatcherGroup
-}
-
 // BatcherClient is a client for building JavaScript packages.
 type BatcherClient struct {
 	d *deps.Deps
@@ -206,12 +197,13 @@ type BatcherClient struct {
 	createClient *create.Client
 	buildClient  *BuildClient
 
-	bundlesCache *maps.Cache[string, BatchPackage]
+	batcherStore *maps.Cache[string, js.Batcher]
+	bundlesStore *maps.Cache[string, js.BatchPackage]
 }
 
 // New creates a new Batcher with the given ID.
 // This will be typically created once and reused across rebuilds.
-func (c *BatcherClient) New(id string) (Batcher, error) {
+func (c *BatcherClient) New(id string) (js.Batcher, error) {
 	var initErr error
 	c.once.Do(func() {
 		// We should fix the initialization order here (or use the Go template package directly), but we need to wait
@@ -288,6 +280,10 @@ func (c *BatcherClient) New(id string) (Batcher, error) {
 	return b, nil
 }
 
+func (c *BatcherClient) Store() *maps.Cache[string, js.Batcher] {
+	return c.batcherStore
+}
+
 func (c *BatcherClient) buildBatchGroup(ctx context.Context, t *batchGroupTemplateContext) (resource.Resource, string, error) {
 	var buf bytes.Buffer
 
@@ -304,18 +300,6 @@ func (c *BatcherClient) buildBatchGroup(ctx context.Context, t *batchGroupTempla
 	return r, s, nil
 }
 
-// BatcherGroup is a group of scripts and instances.
-type BatcherGroup interface {
-	Instance(sid, iid string) OptionsSetter
-	Runner(id string) OptionsSetter
-	Script(id string) OptionsSetter
-}
-
-// OptionsSetter is used to set options for a batch, script or instance.
-type OptionsSetter interface {
-	SetOptions(map[string]any) string
-}
-
 // Package holds a group of JavaScript resources.
 type Package struct {
 	id string
@@ -353,9 +337,9 @@ type batcher struct {
 }
 
 // Build builds the batch if not already built or if it's stale.
-func (b *batcher) Build(ctx context.Context) (BatchPackage, error) {
+func (b *batcher) Build(ctx context.Context) (js.BatchPackage, error) {
 	key := dynacache.CleanKey(b.id + ".js")
-	p, err := b.client.bundlesCache.GetOrCreate(key, func() (BatchPackage, error) {
+	p, err := b.client.bundlesStore.GetOrCreate(key, func() (js.BatchPackage, error) {
 		return b.build(ctx)
 	})
 	if err != nil {
@@ -364,11 +348,11 @@ func (b *batcher) Build(ctx context.Context) (BatchPackage, error) {
 	return p, nil
 }
 
-func (b *batcher) Config(ctx context.Context) OptionsSetter {
+func (b *batcher) Config(ctx context.Context) js.OptionsSetter {
 	return b.configOptions.Get(b.buildCount)
 }
 
-func (b *batcher) Group(ctx context.Context, id string) BatcherGroup {
+func (b *batcher) Group(ctx context.Context, id string) js.BatcherGroup {
 	if err := ValidateBatchID(id, false); err != nil {
 		panic(err)
 	}
@@ -419,7 +403,7 @@ func (b *batcher) isStale() bool {
 	return false
 }
 
-func (b *batcher) build(ctx context.Context) (BatchPackage, error) {
+func (b *batcher) build(ctx context.Context) (js.BatchPackage, error) {
 	b.mu.Lock()
 	defer b.mu.Unlock()
 	defer func() {
@@ -463,6 +447,8 @@ func (b *batcher) doBuild(ctx context.Context) (*Package, error) {
 		pathGroup:             maps.NewCache[string, string](),
 	}
 
+	multihostBasePaths := b.client.d.ResourceSpec.MultihostTargetBasePaths
+
 	// Entry points passed to ESBuid.
 	var entryPoints []string
 	addResource := func(group, pth string, r resource.Resource, isResult bool) {
@@ -701,15 +687,36 @@ func (b *batcher) doBuild(ctx context.Context) (*Package, error) {
 
 		if !handled {
 			//  Copy to destination.
-			p := strings.TrimPrefix(o.Path, outDir)
-			targetFilename := filepath.Join(b.id, p)
-			fs := b.client.d.BaseFs.PublishFs
-			if err := fs.MkdirAll(filepath.Dir(targetFilename), 0o777); err != nil {
-				return nil, fmt.Errorf("failed to create dir %q: %w", targetFilename, err)
+			// In a multihost setup, we will have multiple targets.
+			var targetFilenames []string
+			if len(multihostBasePaths) > 0 {
+				for _, base := range multihostBasePaths {
+					p := strings.TrimPrefix(o.Path, outDir)
+					targetFilename := filepath.Join(base, b.id, p)
+					targetFilenames = append(targetFilenames, targetFilename)
+				}
+			} else {
+				p := strings.TrimPrefix(o.Path, outDir)
+				targetFilename := filepath.Join(b.id, p)
+				targetFilenames = append(targetFilenames, targetFilename)
 			}
 
-			if err := afero.WriteFile(fs, targetFilename, o.Contents, 0o666); err != nil {
-				return nil, fmt.Errorf("failed to write to %q: %w", targetFilename, err)
+			fs := b.client.d.BaseFs.PublishFs
+
+			if err := func() error {
+				fw, err := helpers.OpenFilesForWriting(fs, targetFilenames...)
+				if err != nil {
+					return err
+				}
+				defer fw.Close()
+
+				fr := bytes.NewReader(o.Contents)
+
+				_, err = io.Copy(fw, fr)
+
+				return err
+			}(); err != nil {
+				return nil, fmt.Errorf("failed to copy to %q: %w", targetFilenames, err)
 			}
 		}
 	}
@@ -845,7 +852,7 @@ type optionsGetSetter[K, C any] interface {
 	Key() K
 	Reset()
 
-	Get(uint32) OptionsSetter
+	Get(uint32) js.OptionsSetter
 	isStale() bool
 	currPrev() (map[string]any, map[string]any)
 }
@@ -975,7 +982,7 @@ func (b *scriptGroup) IdentifierBase() string {
 	return b.id
 }
 
-func (s *scriptGroup) Instance(sid, id string) OptionsSetter {
+func (s *scriptGroup) Instance(sid, id string) js.OptionsSetter {
 	if err := ValidateBatchID(sid, false); err != nil {
 		panic(err)
 	}
@@ -1014,7 +1021,7 @@ func (g *scriptGroup) Reset() {
 	}
 }
 
-func (s *scriptGroup) Runner(id string) OptionsSetter {
+func (s *scriptGroup) Runner(id string) js.OptionsSetter {
 	if err := ValidateBatchID(id, false); err != nil {
 		panic(err)
 	}
@@ -1043,7 +1050,7 @@ func (s *scriptGroup) Runner(id string) OptionsSetter {
 	return s.runnersOptions[sid].Get(s.b.buildCount)
 }
 
-func (s *scriptGroup) Script(id string) OptionsSetter {
+func (s *scriptGroup) Script(id string) js.OptionsSetter {
 	if err := ValidateBatchID(id, false); err != nil {
 		panic(err)
 	}