Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Ignore hidden files in batch #617

Merged
merged 6 commits into from
Aug 5, 2024
Merged

Ignore hidden files in batch #617

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
3130f96
Ignore hidden files in batch
DanielCosme Jun 14, 2024
fa1cd88
Allow for hidden directories
DanielCosme Jun 14, 2024
840e405
Clarify comment.
DanielCosme Jun 14, 2024
35ad88e
Remove hidden files when batch is in transferDir
DanielCosme Aug 2, 2024
cc4b936
Add unit tests
DanielCosme Aug 2, 2024
72dee5f
Improve uni test
DanielCosme Aug 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions internal/batch/workflow.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,10 @@
return nil // Keep walking.
}

if strings.HasPrefix(filepath.Base(path), ".") && !entry.IsDir() {
return nil // Don't process hidden files as SIPs

Check warning on line 93 in internal/batch/workflow.go

View check run for this annotation

Codecov / codecov/patch

internal/batch/workflow.go#L93 

Added line #L93 was not covered by tests
}

req := collection.ProcessingWorkflowRequest{
BatchDir: filepath.Dir(path),
Key: entry.Name(),
Expand Down
22 changes: 22 additions & 0 deletions internal/workflow/activities/bundle.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
"context"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
Expand Down Expand Up @@ -61,6 +62,11 @@
res.FullPath = filepath.Join(params.BatchDir, params.Key)
// This makes the workflow not to delete the original content in the transfer directory
res.FullPathBeforeStrip = ""
if params.ExcludeHiddenFiles {
if err := removeHiddenFiles(res.FullPath); err != nil {
return nil, temporal.NewNonRetryableError(fmt.Errorf("failed to remove hidden files: %w", err))

Check warning on line 67 in internal/workflow/activities/bundle.go

View check run for this annotation

Codecov / codecov/patch

internal/workflow/activities/bundle.go#L67 

Added line #L67 was not covered by tests
}
}
} else {
src := filepath.Join(params.BatchDir, params.Key)
dst := params.TransferDir
Expand Down Expand Up @@ -312,3 +318,19 @@

return nil
}

func removeHiddenFiles(path string) error {
return filepath.WalkDir(path, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err

Check warning on line 325 in internal/workflow/activities/bundle.go

View check run for this annotation

Codecov / codecov/patch

internal/workflow/activities/bundle.go#L325 

Added line #L325 was not covered by tests
}
info, err := d.Info()
if err != nil {
return err

Check warning on line 329 in internal/workflow/activities/bundle.go

View check run for this annotation

Codecov / codecov/patch

internal/workflow/activities/bundle.go#L329 

Added line #L329 was not covered by tests
}
if strings.HasPrefix(info.Name(), ".") {
return os.Remove(path)
}
return nil
})
}
45 changes: 45 additions & 0 deletions internal/workflow/activities/bundle_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,51 @@ func TestBundleActivity(t *testing.T) {
),
)
})

t.Run("Remove hidden files when BatchDir is a subfolder of the TransferDir", func(t *testing.T) {
activity := NewBundleActivity()
ts := &temporalsdk_testsuite.WorkflowTestSuite{}
env := ts.NewTestActivityEnvironment()
env.RegisterActivity(activity.Execute)

transferDir := fs.NewDir(t, "enduro",
fs.WithDir("batch-folder",
fs.WithDir(
"sip",
fs.WithFile("foobar.txt", "Hello world!\n"),
fs.WithFile(".hidden", ""),
),
),
)
batchDir := transferDir.Join("batch-folder")
sipSourceDir := transferDir.Join("batch-folder", "sip")

fut, err := env.ExecuteActivity(activity.Execute, &BundleActivityParams{
ExcludeHiddenFiles: true,
IsDir: true,
TransferDir: transferDir.Path(),
BatchDir: batchDir,
Key: "sip",
})
assert.NilError(t, err)

res := BundleActivityResult{}
assert.NilError(t, fut.Get(&res))
assert.Assert(t,
fs.Equal(
sipSourceDir,
fs.Expected(t,
// .hidden is not expected because ExcludeHiddenFiles is enabled.
fs.WithFile("foobar.txt", "Hello world!\n"),
fs.MatchAnyFileMode,
),
),
)
assert.DeepEqual(t, res.FullPath, sipSourceDir)
rePath, err := filepath.Rel(transferDir.Path(), sipSourceDir)
assert.NilError(t, err)
assert.DeepEqual(t, res.RelPath, rePath)
})
}

func TestUnbag(t *testing.T) {
Expand Down
Loading