Skip to content

Commit

Permalink
refactor, introduce cacheDependencyPathToProjectsDirectories
Browse files Browse the repository at this point in the history 
it is necessary for the next PR related yarn optimization
  • Loading branch information
@dsame
dsame committed May 19, 2023
1 parent b779602 commit 47381ae
Show file tree
Hide file tree
Showing 5 changed files with 293 additions and 81 deletions.
60 changes: 46 additions & 14 deletions __tests__/cache-save.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,9 +118,14 @@ describe('run', () => {
expect(getStateSpy).toHaveBeenCalledTimes(2);
expect(getCommandOutputSpy).toHaveBeenCalledTimes(2);
expect(debugSpy).toHaveBeenCalledWith(
'yarn path is /some/random/path/yarn1 (derived from cache-dependency-path: "")'
'Project directory "." derived from cache-dependency-path: ""'
);
expect(debugSpy).toHaveBeenCalledWith(
'Consumed yarn version is 1.2.3 (working dir: ".")'
);
expect(debugSpy).toHaveBeenCalledWith(
'yarn\'s cache folder "/some/random/path/yarn1" configured for the directory "."'
);
expect(debugSpy).toHaveBeenCalledWith('Consumed yarn version is 1.2.3');
expect(infoSpy).toHaveBeenCalledWith(
`Cache hit occurred on the primary key ${yarnFileHash}, not saving cache.`
);
Expand All @@ -140,9 +145,14 @@ describe('run', () => {
expect(getStateSpy).toHaveBeenCalledTimes(2);
expect(getCommandOutputSpy).toHaveBeenCalledTimes(2);
expect(debugSpy).toHaveBeenCalledWith(
'yarn path is /some/random/path/yarn2 (derived from cache-dependency-path: "")'
'Project directory "." derived from cache-dependency-path: ""'
);
expect(debugSpy).toHaveBeenCalledWith(
'Consumed yarn version is 2.2.3 (working dir: ".")'
);
expect(debugSpy).toHaveBeenCalledWith(
'yarn\'s cache folder "/some/random/path/yarn2" configured for the directory "."'
);
expect(debugSpy).toHaveBeenCalledWith('Consumed yarn version is 2.2.3');
expect(infoSpy).toHaveBeenCalledWith(
`Cache hit occurred on the primary key ${yarnFileHash}, not saving cache.`
);
Expand All @@ -159,7 +169,9 @@ describe('run', () => {
expect(getInputSpy).toHaveBeenCalled();
expect(getStateSpy).toHaveBeenCalledTimes(2);
expect(getCommandOutputSpy).toHaveBeenCalledTimes(1);
expect(debugSpy).toHaveBeenCalledWith(`npm path is ${commonPath}/npm`);
expect(debugSpy).toHaveBeenCalledWith(
`npm's cache folder "${commonPath}/npm" configured for the root directory`
);
expect(infoSpy).toHaveBeenCalledWith(
`Cache hit occurred on the primary key ${npmFileHash}, not saving cache.`
);
Expand All @@ -176,7 +188,9 @@ describe('run', () => {
expect(getInputSpy).toHaveBeenCalled();
expect(getStateSpy).toHaveBeenCalledTimes(2);
expect(getCommandOutputSpy).toHaveBeenCalledTimes(1);
expect(debugSpy).toHaveBeenCalledWith(`pnpm path is ${commonPath}/pnpm`);
expect(debugSpy).toHaveBeenCalledWith(
`pnpm's cache folder "${commonPath}/pnpm" configured for the root directory`
);
expect(infoSpy).toHaveBeenCalledWith(
`Cache hit occurred on the primary key ${pnpmFileHash}, not saving cache.`
);
Expand Down Expand Up @@ -204,9 +218,14 @@ describe('run', () => {
expect(getStateSpy).toHaveBeenCalledTimes(2);
expect(getCommandOutputSpy).toHaveBeenCalledTimes(2);
expect(debugSpy).toHaveBeenCalledWith(
'yarn path is /some/random/path/yarn1 (derived from cache-dependency-path: "")'
'Project directory "." derived from cache-dependency-path: ""'
);
expect(debugSpy).toHaveBeenCalledWith(
'Consumed yarn version is 1.2.3 (working dir: ".")'
);
expect(debugSpy).toHaveBeenCalledWith(
'yarn\'s cache folder "/some/random/path/yarn1" configured for the directory "."'
);
expect(debugSpy).toHaveBeenCalledWith('Consumed yarn version is 1.2.3');
expect(infoSpy).not.toHaveBeenCalledWith(
`Cache hit occurred on the primary key ${yarnFileHash}, not saving cache.`
);
Expand Down Expand Up @@ -236,9 +255,14 @@ describe('run', () => {
expect(getStateSpy).toHaveBeenCalledTimes(2);
expect(getCommandOutputSpy).toHaveBeenCalledTimes(2);
expect(debugSpy).toHaveBeenCalledWith(
'yarn path is /some/random/path/yarn2 (derived from cache-dependency-path: "")'
'Project directory "." derived from cache-dependency-path: ""'
);
expect(debugSpy).toHaveBeenCalledWith(
'Consumed yarn version is 2.2.3 (working dir: ".")'
);
expect(debugSpy).toHaveBeenCalledWith(
'yarn\'s cache folder "/some/random/path/yarn2" configured for the directory "."'
);
expect(debugSpy).toHaveBeenCalledWith('Consumed yarn version is 2.2.3');
expect(infoSpy).not.toHaveBeenCalledWith(
`Cache hit occurred on the primary key ${yarnFileHash}, not saving cache.`
);
Expand All @@ -265,7 +289,9 @@ describe('run', () => {
expect(getInputSpy).toHaveBeenCalled();
expect(getStateSpy).toHaveBeenCalledTimes(2);
expect(getCommandOutputSpy).toHaveBeenCalledTimes(1);
expect(debugSpy).toHaveBeenCalledWith(`npm path is ${commonPath}/npm`);
expect(debugSpy).toHaveBeenCalledWith(
`npm's cache folder "${commonPath}/npm" configured for the root directory`
);
expect(infoSpy).not.toHaveBeenCalledWith(
`Cache hit occurred on the primary key ${npmFileHash}, not saving cache.`
);
Expand All @@ -292,7 +318,9 @@ describe('run', () => {
expect(getInputSpy).toHaveBeenCalled();
expect(getStateSpy).toHaveBeenCalledTimes(2);
expect(getCommandOutputSpy).toHaveBeenCalledTimes(1);
expect(debugSpy).toHaveBeenCalledWith(`pnpm path is ${commonPath}/pnpm`);
expect(debugSpy).toHaveBeenCalledWith(
`pnpm's cache folder "${commonPath}/pnpm" configured for the root directory`
);
expect(infoSpy).not.toHaveBeenCalledWith(
`Cache hit occurred on the primary key ${pnpmFileHash}, not saving cache.`
);
Expand Down Expand Up @@ -322,7 +350,9 @@ describe('run', () => {
expect(getInputSpy).toHaveBeenCalled();
expect(getStateSpy).toHaveBeenCalledTimes(2);
expect(getCommandOutputSpy).toHaveBeenCalledTimes(1);
expect(debugSpy).toHaveBeenCalledWith(`npm path is ${commonPath}/npm`);
expect(debugSpy).toHaveBeenCalledWith(
`npm's cache folder "${commonPath}/npm" configured for the root directory`
);
expect(infoSpy).not.toHaveBeenCalledWith(
`Cache hit occurred on the primary key ${npmFileHash}, not saving cache.`
);
Expand Down Expand Up @@ -352,7 +382,9 @@ describe('run', () => {
expect(getInputSpy).toHaveBeenCalled();
expect(getStateSpy).toHaveBeenCalledTimes(2);
expect(getCommandOutputSpy).toHaveBeenCalledTimes(1);
expect(debugSpy).toHaveBeenCalledWith(`npm path is ${commonPath}/npm`);
expect(debugSpy).toHaveBeenCalledWith(
`npm's cache folder "${commonPath}/npm" configured for the root directory`
);
expect(infoSpy).not.toHaveBeenCalledWith(
`Cache hit occurred on the primary key ${npmFileHash}, not saving cache.`
);
Expand Down
41 changes: 32 additions & 9 deletions __tests__/cache-utils.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@ import {
isCacheFeatureAvailable,
supportedPackageManagers,
getCommandOutput,
expandCacheDependencyPath
expandCacheDependencyPath,
expandedPatternsMemoized
} from '../src/cache-utils';
import fs from 'fs';
import * as cacheUtils from '../src/cache-utils';
Expand Down Expand Up @@ -104,6 +105,10 @@ describe('cache-utils', () => {
(pattern: string): Promise<Globber> =>
MockGlobber.create(['/foo', '/bar'])
);

Object.keys(expandedPatternsMemoized).forEach(
key => delete expandedPatternsMemoized[key]
);
});

afterEach(() => {
Expand Down Expand Up @@ -194,13 +199,36 @@ two
[supportedPackageManagers.yarn, '/dir/file.lock'],
[supportedPackageManagers.yarn, '/**/file.lock']
])(
'getCacheDirectoriesPaths should return empty array of folder in case of error',
'getCacheDirectoriesPaths should throw for getCommandOutput returning empty',
async (packageManagerInfo, cacheDependency) => {
getCommandOutputSpy.mockImplementation((command: string) =>
// return empty string to indicate getCacheFolderPath failed
// --version still works
command.includes('version') ? '1.' : ''
);

await expect(
cacheUtils.getCacheDirectoriesPaths(
packageManagerInfo,
cacheDependency
)
).rejects.toThrow(); //'Could not get cache folder path for /dir');
}
);

it.each([
[supportedPackageManagers.npm, ''],
[supportedPackageManagers.npm, '/dir/file.lock'],
[supportedPackageManagers.npm, '/**/file.lock'],
[supportedPackageManagers.pnpm, ''],
[supportedPackageManagers.pnpm, '/dir/file.lock'],
[supportedPackageManagers.pnpm, '/**/file.lock'],
[supportedPackageManagers.yarn, ''],
[supportedPackageManagers.yarn, '/dir/file.lock'],
[supportedPackageManagers.yarn, '/**/file.lock']
])(
'getCacheDirectoriesPaths should throw in case of having not directories',
async (packageManagerInfo, cacheDependency) => {
lstatSpy.mockImplementation(arg => ({
isDirectory: () => false
}));
Expand Down Expand Up @@ -248,9 +276,8 @@ two
}
);

// TODO: by design - glob is not expected to return duplicates so 3 patterns do not collapse to 2
it.each(['1.1.1', '2.2.2'])(
'getCacheDirectoriesPaths yarn v%s should return 3 dirs with globbed cacheDependency expanding to duplicates',
'getCacheDirectoriesPaths yarn v%s should return 2 dirs with globbed cacheDependency expanding to duplicates',
async version => {
let dirNo = 1;
getCommandOutputSpy.mockImplementation((command: string) =>
Expand All @@ -269,11 +296,7 @@ two
supportedPackageManagers.yarn,
'/tmp/**/file'
);
expect(dirs).toEqual([
`file_${version}_1`,
`file_${version}_2`,
`file_${version}_3`
]);
expect(dirs).toEqual([`file_${version}_1`, `file_${version}_2`]);
}
);

Expand Down
76 changes: 61 additions & 15 deletions dist/cache-save/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60434,7 +60434,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", ({ value: true }));
exports.isCacheFeatureAvailable = exports.isGhes = exports.getCacheDirectoriesPaths = exports.expandCacheDependencyPath = exports.getPackageManagerInfo = exports.getCommandOutputGuarded = exports.getCommandOutput = exports.supportedPackageManagers = exports.yarn2GetCacheFolderCommand = exports.yarn1GetCacheFolderCommand = exports.pnpmGetCacheFolderCommand = exports.npmGetCacheFolderCommand = void 0;
exports.isCacheFeatureAvailable = exports.isGhes = exports.getCacheDirectoriesPaths = exports.expandCacheDependencyPath = exports.expandedPatternsMemoized = exports.getPackageManagerInfo = exports.getCommandOutputGuarded = exports.getCommandOutput = exports.supportedPackageManagers = exports.yarn2GetCacheFolderCommand = exports.yarn1GetCacheFolderCommand = exports.pnpmGetCacheFolderCommand = exports.npmGetCacheFolderCommand = void 0;
const core = __importStar(__nccwpck_require__(2186));
const exec = __importStar(__nccwpck_require__(1514));
const cache = __importStar(__nccwpck_require__(7799));
Expand Down Expand Up @@ -60462,7 +60462,7 @@ exports.supportedPackageManagers = {
lockFilePatterns: ['yarn.lock'],
getCacheFolderPath: (projectDir) => __awaiter(void 0, void 0, void 0, function* () {
const yarnVersion = yield exports.getCommandOutputGuarded(`yarn --version`, 'Could not retrieve version of yarn', projectDir);
core.debug(`Consumed yarn version is ${yarnVersion}`);
core.debug(`Consumed yarn version is ${yarnVersion} (working dir: "${projectDir}")`);
const stdOut = yarnVersion.startsWith('1.')
? yield exports.getCommandOutput(exports.yarn1GetCacheFolderCommand, projectDir)
: yield exports.getCommandOutput(exports.yarn2GetCacheFolderCommand, projectDir);
Expand Down Expand Up @@ -60507,10 +60507,27 @@ const getPackageManagerInfo = (packageManager) => __awaiter(void 0, void 0, void
}
});
exports.getPackageManagerInfo = getPackageManagerInfo;
exports.expandedPatternsMemoized = {};
/**
* Wrapper around `glob.create(pattern).glob()` with the memoization
* @param pattern is expected to be a globed path
* @return list of files or directories expanded from glob
*/
const globPatternToArray = (pattern) => __awaiter(void 0, void 0, void 0, function* () {
const memoized = exports.expandedPatternsMemoized[pattern];
if (memoized)
return Promise.resolve(memoized);
const globber = yield glob.create(pattern);
return globber.glob();
const expanded = yield globber.glob();
exports.expandedPatternsMemoized[pattern] = expanded;
return expanded;
});
/**
* Expands (converts) the string input `cache-dependency-path` to list of files' paths
* First it breaks the input by new lines and then expand glob patterns if any
* @param cacheDependencyPath - either a single string or multiline string with possible glob patterns
* @return list of files on which the cache depends
*/
const expandCacheDependencyPath = (cacheDependencyPath) => __awaiter(void 0, void 0, void 0, function* () {
const multilinePaths = cacheDependencyPath
.split(/\r?\n/)
Expand All @@ -60521,26 +60538,55 @@ const expandCacheDependencyPath = (cacheDependencyPath) => __awaiter(void 0, voi
return expandedPaths.length === 0 ? [''] : expandedPaths.flat();
});
exports.expandCacheDependencyPath = expandCacheDependencyPath;
const cacheDependencyPathToCacheFolderPath = (packageManagerInfo, cacheDependencyPath) => __awaiter(void 0, void 0, void 0, function* () {
const cacheDependencyPathDirectory = path_1.default.dirname(cacheDependencyPath);
const cacheFolderPath = fs_1.default.existsSync(cacheDependencyPathDirectory) &&
fs_1.default.lstatSync(cacheDependencyPathDirectory).isDirectory()
? yield packageManagerInfo.getCacheFolderPath(cacheDependencyPathDirectory)
: yield packageManagerInfo.getCacheFolderPath();
core.debug(`${packageManagerInfo.name} path is ${cacheFolderPath} (derived from cache-dependency-path: "${cacheDependencyPath}")`);
/**
* Converts dependency file to the directory it resides in and ensures the directory exists
* @param cacheDependencyPath - file name
* @return either directory containing file or null
*/
const cacheDependencyPathToProjectDirectory = (cacheDependencyPath) => {
const projectDirectory = path_1.default.dirname(cacheDependencyPath);
if (fs_1.default.existsSync(projectDirectory) &&
fs_1.default.lstatSync(projectDirectory).isDirectory()) {
core.debug(`Project directory "${projectDirectory}" derived from cache-dependency-path: "${cacheDependencyPath}"`);
return projectDirectory;
}
else {
core.debug(`No project directory found for cache-dependency-path: "${cacheDependencyPath}", will be skipped`);
return null;
}
};
/**
* Expands (converts) the string input `cache-dependency-path` to list of directories that
* may be project roots
* @param cacheDependencyPath
* @return list of directories and possible
*/
const cacheDependencyPathToProjectsDirectories = (cacheDependencyPath) => __awaiter(void 0, void 0, void 0, function* () {
const cacheDependenciesPaths = yield exports.expandCacheDependencyPath(cacheDependencyPath);
const existingDirectories = cacheDependenciesPaths
.map(cacheDependencyPath => cacheDependencyPathToProjectDirectory(cacheDependencyPath))
.filter(path => path !== null);
if (existingDirectories.length === 0)
throw Error('No existing directories found containing `cache-dependency-path`="${cacheDependencyPath}"');
// uniq
return existingDirectories.filter((cachePath, i, result) => cachePath != null && result.indexOf(cachePath) === i);
});
const projectDirectoryToCacheFolderPath = (packageManagerInfo, projectDirectory) => __awaiter(void 0, void 0, void 0, function* () {
const cacheFolderPath = yield packageManagerInfo.getCacheFolderPath(projectDirectory);
core.debug(`${packageManagerInfo.name}'s cache folder "${cacheFolderPath}" configured for the directory "${projectDirectory}"`);
return cacheFolderPath;
});
const cacheDependenciesPathsToCacheFoldersPaths = (packageManagerInfo, cacheDependenciesPaths) => __awaiter(void 0, void 0, void 0, function* () {
const cacheFoldersPaths = yield Promise.all(cacheDependenciesPaths.map(cacheDependencyPath => cacheDependencyPathToCacheFolderPath(packageManagerInfo, cacheDependencyPath)));
const projectDirectoriesToCacheFoldersPaths = (packageManagerInfo, projectDirectories) => __awaiter(void 0, void 0, void 0, function* () {
const cacheFoldersPaths = yield Promise.all(projectDirectories.map(projectDirectory => projectDirectoryToCacheFolderPath(packageManagerInfo, projectDirectory)));
return cacheFoldersPaths.filter((cachePath, i, result) => result.indexOf(cachePath) === i);
});
const cacheDependencyPathToCacheFoldersPaths = (packageManagerInfo, cacheDependencyPath) => __awaiter(void 0, void 0, void 0, function* () {
const cacheDependenciesPaths = yield exports.expandCacheDependencyPath(cacheDependencyPath);
return cacheDependenciesPathsToCacheFoldersPaths(packageManagerInfo, cacheDependenciesPaths);
const projectDirectories = yield cacheDependencyPathToProjectsDirectories(cacheDependencyPath);
return projectDirectoriesToCacheFoldersPaths(packageManagerInfo, projectDirectories);
});
const cacheFoldersPathsForRoot = (packageManagerInfo) => __awaiter(void 0, void 0, void 0, function* () {
const cacheFolderPath = yield packageManagerInfo.getCacheFolderPath();
core.debug(`${packageManagerInfo.name} path is ${cacheFolderPath}`);
core.debug(`${packageManagerInfo.name}'s cache folder "${cacheFolderPath}" configured for the root directory`);
return [cacheFolderPath];
});
const getCacheDirectoriesPaths = (packageManagerInfo, cacheDependencyPath) => __awaiter(void 0, void 0, void 0, function* () {
Expand Down
Loading

0 comments on commit 47381ae

Please sign in to comment.