feat: implement Julia parser (#219)

aquasecurity · Nov 22, 2023 · fc7f2b4 · fc7f2b4
1 parent 8322cc2
commit fc7f2b4
Show file tree

Hide file tree

Showing 10 changed files with 512 additions and 0 deletions.
diff --git a/pkg/julia/manifest/naive_pkg_parser.go b/pkg/julia/manifest/naive_pkg_parser.go
@@ -0,0 +1,68 @@
+package julia
+
+import (
+	"bufio"
+	"io"
+	"strings"
+)
+
+type pkgPosition struct {
+	start int
+	end   int
+}
+type minPkg struct {
+	uuid     string
+	version  string
+	position pkgPosition
+}
+
+func (pkg *minPkg) setEndPositionIfEmpty(n int) {
+	if pkg.position.end == 0 {
+		pkg.position.end = n
+	}
+}
+
+type naivePkgParser struct {
+	r io.Reader
+}
+
+func (parser *naivePkgParser) parse() map[string]pkgPosition {
+	var currentPkg minPkg = minPkg{}
+	var idx = make(map[string]pkgPosition, 0)
+
+	scanner := bufio.NewScanner(parser.r)
+	lineNum := 1
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.HasPrefix(strings.TrimSpace(line), "[") {
+			if currentPkg.uuid != "" {
+				currentPkg.setEndPositionIfEmpty(lineNum - 1)
+				idx[currentPkg.uuid] = currentPkg.position
+			}
+			currentPkg = minPkg{}
+			currentPkg.position.start = lineNum
+
+		} else if strings.HasPrefix(strings.TrimSpace(line), "uuid =") {
+			currentPkg.uuid = propertyValue(line)
+		} else if strings.HasPrefix(strings.TrimSpace(line), "version =") {
+			currentPkg.version = propertyValue(line)
+		} else if strings.TrimSpace(line) == "" {
+			currentPkg.setEndPositionIfEmpty(lineNum - 1)
+		}
+
+		lineNum++
+	}
+	// add last item
+	if currentPkg.uuid != "" {
+		currentPkg.setEndPositionIfEmpty(lineNum - 1)
+		idx[currentPkg.uuid] = currentPkg.position
+	}
+	return idx
+}
+func propertyValue(line string) string {
+	parts := strings.Split(line, "=")
+	if len(parts) == 2 {
+		return strings.Trim(parts[1], ` "`)
+	}
+	return ""
+}
diff --git a/pkg/julia/manifest/parse.go b/pkg/julia/manifest/parse.go
@@ -0,0 +1,161 @@
+package julia
+
+import (
+	"io"
+	"sort"
+
+	"github.com/BurntSushi/toml"
+	dio "github.com/aquasecurity/go-dep-parser/pkg/io"
+	"github.com/aquasecurity/go-dep-parser/pkg/types"
+
+	"golang.org/x/exp/maps"
+	"golang.org/x/xerrors"
+)
+
+type primitiveManifest struct {
+	JuliaVersion   string                           `toml:"julia_version"`
+	ManifestFormat string                           `toml:"manifest_format"`
+	Dependencies   map[string][]primitiveDependency `toml:"deps"` // e.g. [[deps.Foo]]
+}
+
+type primitiveDependency struct {
+	Dependencies toml.Primitive `toml:"deps"` // by name. e.g. deps = ["Foo"] or [deps.Foo.deps]
+	UUID         string         `toml:"uuid"`
+	Version      string         `toml:"version"` // not specified for stdlib packages, which are of the Julia version
+	DependsOn    []string       `toml:"-"`       // list of dependent UUID's.
+}
+
+type Parser struct{}
+
+func NewParser() types.Parser {
+	return &Parser{}
+}
+
+func (p *Parser) Parse(r dio.ReadSeekerAt) ([]types.Library, []types.Dependency, error) {
+	var oldDeps map[string][]primitiveDependency
+	var primMan primitiveManifest
+	var manMetadata toml.MetaData
+	decoder := toml.NewDecoder(r)
+	// Try to read the old Manifest format. If that fails, try the new format.
+	if _, err := decoder.Decode(&oldDeps); err != nil {
+		if _, err = r.Seek(0, io.SeekStart); err != nil {
+			return nil, nil, xerrors.Errorf("seek error: %w", err)
+		}
+		if manMetadata, err = decoder.Decode(&primMan); err != nil {
+			return nil, nil, xerrors.Errorf("decode error: %w", err)
+		}
+	}
+
+	// We can't know the Julia version on an old manifest.
+	// All newer manifests include a manifest version and a julia version.
+	if primMan.ManifestFormat == "" {
+		primMan = primitiveManifest{
+			JuliaVersion: "unknown",
+			Dependencies: oldDeps,
+		}
+	}
+
+	man, err := decodeManifest(&primMan, &manMetadata)
+	if err != nil {
+		return nil, nil, xerrors.Errorf("unable to decode manifest dependencies: %w", err)
+	}
+
+	if _, err := r.Seek(0, io.SeekStart); err != nil {
+		return nil, nil, xerrors.Errorf("seek error: %w", err)
+	}
+
+	// naive parser to get line numbers
+	pkgParser := naivePkgParser{r: r}
+	lineNumIdx := pkgParser.parse()
+
+	var libs []types.Library
+	var deps []types.Dependency
+	for name, manifestDeps := range man.Dependencies {
+		for _, manifestDep := range manifestDeps {
+			version := depVersion(&manifestDep, man.JuliaVersion)
+			pkgID := manifestDep.UUID
+			lib := types.Library{
+				ID:      pkgID,
+				Name:    name,
+				Version: version,
+			}
+			if pos, ok := lineNumIdx[manifestDep.UUID]; ok {
+				lib.Locations = []types.Location{{StartLine: pos.start, EndLine: pos.end}}
+			}
+
+			libs = append(libs, lib)
+
+			if len(manifestDep.DependsOn) > 0 {
+				deps = append(deps, types.Dependency{
+					ID:        pkgID,
+					DependsOn: manifestDep.DependsOn,
+				})
+			}
+		}
+	}
+	sort.Sort(types.Libraries(libs))
+	sort.Sort(types.Dependencies(deps))
+	return libs, deps, nil
+}
+
+// Returns the effective version of the `dep`.
+// stdlib packages do not have a version in the manifest because they are packaged with julia itself
+func depVersion(dep *primitiveDependency, juliaVersion string) string {
+	if len(dep.Version) == 0 {
+		return juliaVersion
+	}
+	return dep.Version
+}
+
+// Decodes a primitive manifest using the metadata from parse time.
+func decodeManifest(man *primitiveManifest, metadata *toml.MetaData) (*primitiveManifest, error) {
+	// Decode each dependency into the new manifest
+	for depName, primDeps := range man.Dependencies {
+		var newPrimDeps []primitiveDependency
+		for _, primDep := range primDeps {
+			newPrimDep, err := decodeDependency(man, primDep, metadata)
+			if err != nil {
+				return nil, err
+			}
+			newPrimDeps = append(newPrimDeps, newPrimDep)
+		}
+		man.Dependencies[depName] = newPrimDeps
+	}
+
+	return man, nil
+}
+
+// Decodes a primitive dependency using the metadata from parse time.
+func decodeDependency(man *primitiveManifest, dep primitiveDependency, metadata *toml.MetaData) (primitiveDependency, error) {
+	// Try to decode as []string first where the manifest looks like deps = ["A", "B"]
+	var possibleDeps []string
+	err := metadata.PrimitiveDecode(dep.Dependencies, &possibleDeps)
+	if err == nil {
+		var possibleUuids []string
+		for _, depName := range possibleDeps {
+			primDep := man.Dependencies[depName]
+			if len(primDep) > 1 {
+				return primitiveDependency{}, xerrors.Errorf("Dependency %q has invalid format (parsed multiple deps): %s", depName, primDep)
+			}
+			possibleUuids = append(possibleUuids, primDep[0].UUID)
+		}
+		sort.Strings(possibleUuids)
+		dep.DependsOn = possibleUuids
+		return dep, nil
+	}
+
+	// The other possibility is a map where the manifest looks like
+	// [deps.A.deps]
+	// B = "..."
+	var possibleDepsMap map[string]string
+	err = metadata.PrimitiveDecode(dep.Dependencies, &possibleDepsMap)
+	if err == nil {
+		possibleUuids := maps.Values(possibleDepsMap)
+		sort.Strings(possibleUuids)
+		dep.DependsOn = possibleUuids
+		return dep, nil
+	}
+
+	// We don't know what the shape of the data is -- i.e. an invalid manifest
+	return primitiveDependency{}, err
+}
diff --git a/pkg/julia/manifest/parse_test.go b/pkg/julia/manifest/parse_test.go
@@ -0,0 +1,75 @@
+package julia
+
+import (
+	"os"
+	"sort"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/aquasecurity/go-dep-parser/pkg/types"
+)
+
+func TestParse(t *testing.T) {
+	tests := []struct {
+		name     string
+		file     string // Test input file
+		want     []types.Library
+		wantDeps []types.Dependency
+	}{
+		{
+			name:     "Manifest v1.6",
+			file:     "testdata/primary/Manifest_v1.6.toml",
+			want:     juliaV1_6Libs,
+			wantDeps: juliaV1_6Deps,
+		},
+		{
+			name:     "Manifest v1.8",
+			file:     "testdata/primary/Manifest_v1.8.toml",
+			want:     juliaV1_8Libs,
+			wantDeps: juliaV1_8Deps,
+		},
+		{
+			name:     "no deps v1.6",
+			file:     "testdata/no_deps_v1.6/Manifest.toml",
+			want:     nil,
+			wantDeps: nil,
+		},
+		{
+			name:     "no deps v1.9",
+			file:     "testdata/no_deps_v1.9/Manifest.toml",
+			want:     nil,
+			wantDeps: nil,
+		},
+		{
+			name:     "dep extensions v1.9",
+			file:     "testdata/dep_ext_v1.9/Manifest.toml",
+			want:     juliaV1_9DepExtLibs,
+			wantDeps: nil,
+		},
+		{
+			name:     "shadowed dep v1.9",
+			file:     "testdata/shadowed_dep_v1.9/Manifest.toml",
+			want:     juliaV1_9ShadowedDepLibs,
+			wantDeps: juliaV1_9ShadowedDepDeps,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			f, err := os.Open(tt.file)
+			require.NoError(t, err)
+
+			gotLibs, gotDeps, err := NewParser().Parse(f)
+			require.NoError(t, err)
+
+			sort.Sort(types.Libraries(tt.want))
+			assert.Equal(t, tt.want, gotLibs)
+			if tt.wantDeps != nil {
+				sort.Sort(types.Dependencies(tt.wantDeps))
+				assert.Equal(t, tt.wantDeps, gotDeps)
+			}
+		})
+	}
+}
diff --git a/pkg/julia/manifest/parse_testcase.go b/pkg/julia/manifest/parse_testcase.go
@@ -0,0 +1,77 @@
+package julia
+
+import "github.com/aquasecurity/go-dep-parser/pkg/types"
+
+var (
+	juliaV1_6Libs = []types.Library{
+		{ID: "ade2ca70-3891-5945-98fb-dc099432e06a", Name: "Dates", Version: "unknown", Locations: []types.Location{{StartLine: 3, EndLine: 5}}},
+		{ID: "682c06a0-de6a-54ab-a142-c8b1cf79cde6", Name: "JSON", Version: "0.21.4", Locations: []types.Location{{StartLine: 7, EndLine: 11}}},
+		{ID: "a63ad114-7e13-5084-954f-fe012c677804", Name: "Mmap", Version: "unknown", Locations: []types.Location{{StartLine: 13, EndLine: 14}}},
+		{ID: "69de0a69-1ddd-5017-9359-2bf0b02dc9f0", Name: "Parsers", Version: "2.4.2", Locations: []types.Location{{StartLine: 16, EndLine: 20}}},
+		{ID: "de0858da-6303-5e67-8744-51eddeeeb8d7", Name: "Printf", Version: "unknown", Locations: []types.Location{{StartLine: 22, EndLine: 24}}},
+		{ID: "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5", Name: "Unicode", Version: "unknown", Locations: []types.Location{{StartLine: 26, EndLine: 27}}},
+	}
+
+	juliaV1_6Deps = []types.Dependency{
+		{ID: "ade2ca70-3891-5945-98fb-dc099432e06a", DependsOn: []string{"de0858da-6303-5e67-8744-51eddeeeb8d7"}},
+		{ID: "682c06a0-de6a-54ab-a142-c8b1cf79cde6", DependsOn: []string{
+			"4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5",
+			"69de0a69-1ddd-5017-9359-2bf0b02dc9f0",
+			"a63ad114-7e13-5084-954f-fe012c677804",
+			"ade2ca70-3891-5945-98fb-dc099432e06a",
+		}},
+		{ID: "69de0a69-1ddd-5017-9359-2bf0b02dc9f0", DependsOn: []string{"ade2ca70-3891-5945-98fb-dc099432e06a"}},
+		{ID: "de0858da-6303-5e67-8744-51eddeeeb8d7", DependsOn: []string{"4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"}},
+	}
+
+	juliaV1_8Libs = []types.Library{
+		{ID: "ade2ca70-3891-5945-98fb-dc099432e06a", Name: "Dates", Version: "1.8.5", Locations: []types.Location{{StartLine: 7, EndLine: 9}}},
+		{ID: "682c06a0-de6a-54ab-a142-c8b1cf79cde6", Name: "JSON", Version: "0.21.4", Locations: []types.Location{{StartLine: 11, EndLine: 15}}},
+		{ID: "a63ad114-7e13-5084-954f-fe012c677804", Name: "Mmap", Version: "1.8.5", Locations: []types.Location{{StartLine: 17, EndLine: 18}}},
+		{ID: "69de0a69-1ddd-5017-9359-2bf0b02dc9f0", Name: "Parsers", Version: "2.5.10", Locations: []types.Location{{StartLine: 20, EndLine: 24}}},
+		{ID: "aea7be01-6a6a-4083-8856-8a6e6704d82a", Name: "PrecompileTools", Version: "1.1.1", Locations: []types.Location{{StartLine: 26, EndLine: 30}}},
+		{ID: "21216c6a-2e73-6563-6e65-726566657250", Name: "Preferences", Version: "1.4.0", Locations: []types.Location{{StartLine: 32, EndLine: 36}}},
+		{ID: "de0858da-6303-5e67-8744-51eddeeeb8d7", Name: "Printf", Version: "1.8.5", Locations: []types.Location{{StartLine: 38, EndLine: 40}}},
+		{ID: "9a3f8284-a2c9-5f02-9a11-845980a1fd5c", Name: "Random", Version: "1.8.5", Locations: []types.Location{{StartLine: 42, EndLine: 44}}},
+		{ID: "ea8e919c-243c-51af-8825-aaa63cd721ce", Name: "SHA", Version: "0.7.0", Locations: []types.Location{{StartLine: 46, EndLine: 48}}},
+		{ID: "9e88b42a-f829-5b0c-bbe9-9e923198166b", Name: "Serialization", Version: "1.8.5", Locations: []types.Location{{StartLine: 50, EndLine: 51}}},
+		{ID: "fa267f1f-6049-4f14-aa54-33bafae1ed76", Name: "TOML", Version: "1.0.0", Locations: []types.Location{{StartLine: 53, EndLine: 56}}},
+		{ID: "cf7118a7-6976-5b1a-9a39-7adc72f591a4", Name: "UUIDs", Version: "1.8.5", Locations: []types.Location{{StartLine: 58, EndLine: 60}}},
+		{ID: "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5", Name: "Unicode", Version: "1.8.5", Locations: []types.Location{{StartLine: 62, EndLine: 63}}},
+	}
+
+	juliaV1_8Deps = []types.Dependency{
+		{ID: "ade2ca70-3891-5945-98fb-dc099432e06a", DependsOn: []string{"de0858da-6303-5e67-8744-51eddeeeb8d7"}},
+		{ID: "682c06a0-de6a-54ab-a142-c8b1cf79cde6", DependsOn: []string{
+			"4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5",
+			"69de0a69-1ddd-5017-9359-2bf0b02dc9f0",
+			"a63ad114-7e13-5084-954f-fe012c677804",
+			"ade2ca70-3891-5945-98fb-dc099432e06a",
+		}},
+		{ID: "69de0a69-1ddd-5017-9359-2bf0b02dc9f0", DependsOn: []string{
+			"ade2ca70-3891-5945-98fb-dc099432e06a",
+			"aea7be01-6a6a-4083-8856-8a6e6704d82a",
+			"cf7118a7-6976-5b1a-9a39-7adc72f591a4",
+		}},
+		{ID: "aea7be01-6a6a-4083-8856-8a6e6704d82a", DependsOn: []string{"21216c6a-2e73-6563-6e65-726566657250"}},
+		{ID: "21216c6a-2e73-6563-6e65-726566657250", DependsOn: []string{"fa267f1f-6049-4f14-aa54-33bafae1ed76"}},
+		{ID: "de0858da-6303-5e67-8744-51eddeeeb8d7", DependsOn: []string{"4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"}},
+		{ID: "9a3f8284-a2c9-5f02-9a11-845980a1fd5c", DependsOn: []string{"9e88b42a-f829-5b0c-bbe9-9e923198166b", "ea8e919c-243c-51af-8825-aaa63cd721ce"}},
+		{ID: "fa267f1f-6049-4f14-aa54-33bafae1ed76", DependsOn: []string{"ade2ca70-3891-5945-98fb-dc099432e06a"}},
+		{ID: "cf7118a7-6976-5b1a-9a39-7adc72f591a4", DependsOn: []string{"9a3f8284-a2c9-5f02-9a11-845980a1fd5c", "ea8e919c-243c-51af-8825-aaa63cd721ce"}},
+	}
+
+	juliaV1_9DepExtLibs = []types.Library{
+		{ID: "621f4979-c628-5d54-868e-fcf4e3e8185c", Name: "AbstractFFTs", Version: "1.3.1", Locations: []types.Location{{StartLine: 7, EndLine: 10}}},
+	}
+
+	juliaV1_9ShadowedDepLibs = []types.Library{
+		{ID: "ead4f63c-334e-11e9-00e6-e7f0a5f21b60", Name: "A", Version: "1.9.0", Locations: []types.Location{{StartLine: 7, EndLine: 8}}},
+		{ID: "f41f7b98-334e-11e9-1257-49272045fb24", Name: "B", Version: "1.9.0", Locations: []types.Location{{StartLine: 13, EndLine: 14}}},
+		{ID: "edca9bc6-334e-11e9-3554-9595dbb4349c", Name: "B", Version: "1.9.0", Locations: []types.Location{{StartLine: 15, EndLine: 16}}},
+	}
+
+	juliaV1_9ShadowedDepDeps = []types.Dependency{
+		{ID: "ead4f63c-334e-11e9-00e6-e7f0a5f21b60", DependsOn: []string{"f41f7b98-334e-11e9-1257-49272045fb24"}},
+	}
+)