Refactoring packages-1

dotnet · akshita31 · Apr 24, 2018 · Apr 10, 2018 · Apr 10, 2018 · Apr 10, 2018
commit c74aecf639b40566734a3aa8f8c898bb0540c6e6
diff --git a/src/CSharpExtDownloader.ts b/src/CSharpExtDownloader.ts
@@ -4,32 +4,25 @@
  *--------------------------------------------------------------------------------------------*/
 
 import * as util from './common';
-import { GetNetworkConfiguration, defaultPackageManagerFactory, IPackageManagerFactory } from './downloader.helper';
-import { PackageManager, Package } from './packages';
 import { PlatformInformation } from './platform';
 import { PackageInstallation, LogPlatformInfo, InstallationSuccess, InstallationFailure } from './omnisharp/loggingEvents';
 import { EventStream } from './EventStream';
 import { vscode } from './vscodeAdapter';
+import { PackageManager } from './packageManager/PackageManager';
+import { Package } from './packageManager/packages';
 
 /*
  * Class used to download the runtime dependencies of the C# Extension
  */
 export class CSharpExtDownloader {
-    private proxy: string;
-    private strictSSL: boolean;
     private packageManager: PackageManager;
 
     public constructor(
-        vscode: vscode,
+        private vscode: vscode,
         private eventStream: EventStream,
         private packageJSON: any,
-        private platformInfo: PlatformInformation,
-        packageManagerFactory: IPackageManagerFactory = defaultPackageManagerFactory) {
-
-        let networkConfiguration = GetNetworkConfiguration(vscode);
-        this.proxy = networkConfiguration.Proxy;
-        this.strictSSL = networkConfiguration.StrictSSL;
-        this.packageManager = packageManagerFactory(this.platformInfo);
+        private platformInfo: PlatformInformation) {
+        this.packageManager = new PackageManager();
     }
 
     public async installRuntimeDependencies(): Promise<boolean> {
@@ -45,12 +38,10 @@ export class CSharpExtDownloader {
 
             let runTimeDependencies = this.GetRunTimeDependenciesPackages();
 
-            installationStage = 'downloadPackages';
-            let downloadedPackages = await this.packageManager.DownloadPackages(runTimeDependencies, this.eventStream, this.proxy, this.strictSSL);
-
-            installationStage = 'installPackages';
-            await this.packageManager.InstallPackages(downloadedPackages,this.eventStream);
+            installationStage = 'downloadAndInstallPackages';
+            await this.packageManager.DownloadAndInstallPackages(runTimeDependencies, this.vscode, this.platformInfo, this.eventStream);
 
+            // We probably dont need the install.Lock thing now as we are directly testing the package test path
             installationStage = 'touchLockFile';
             await util.touchInstallFile(util.InstallFileType.Lock);
 

diff --git a/src/downloader.helper.ts b/src/downloader.helper.ts
diff --git a/src/omnisharp/OmnisharpDownloader.ts b/src/omnisharp/OmnisharpDownloader.ts
@@ -3,13 +3,12 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { GetNetworkConfiguration, IPackageManagerFactory, defaultPackageManagerFactory } from '../downloader.helper';
 import { GetPackagesFromVersion, GetVersionFilePackage } from './OmnisharpPackageCreator';
-import { PackageManager } from '../packages';
 import { PlatformInformation } from '../platform';
 import { PackageInstallation, LogPlatformInfo, InstallationSuccess, InstallationFailure, InstallationProgress } from './loggingEvents';
 import { EventStream } from '../EventStream';
 import { vscode } from '../vscodeAdapter';
+import { PackageManager } from '../packageManager/PackageManager';
 
 
 export class OmnisharpDownloader {
@@ -18,16 +17,12 @@ export class OmnisharpDownloader {
     private packageManager: PackageManager;
 
     public constructor(
-        vscode: vscode,
+        private vscode: vscode,
         private eventStream: EventStream,
         private packageJSON: any,
-        private platformInfo: PlatformInformation,
-        packageManagerFactory: IPackageManagerFactory = defaultPackageManagerFactory) {
+        private platformInfo: PlatformInformation) {
 
-        let networkConfiguration = GetNetworkConfiguration(vscode);
-        this.proxy = networkConfiguration.Proxy;
-        this.strictSSL = networkConfiguration.StrictSSL;
-        this.packageManager = packageManagerFactory(this.platformInfo);
+        this.packageManager = new PackageManager();
     }
 
     public async DownloadAndInstallOmnisharp(version: string, serverUrl: string, installPath: string) {
@@ -40,12 +35,8 @@ export class OmnisharpDownloader {
             installationStage = 'getPackageInfo';
             let packages = GetPackagesFromVersion(version, this.packageJSON.runtimeDependencies, serverUrl, installPath);
 
-            installationStage = 'downloadPackages';
-
-            let downloadedPackages = await this.packageManager.DownloadPackages(packages, this.eventStream, this.proxy, this.strictSSL);
-
-            installationStage = 'installPackages';
-            await this.packageManager.InstallPackages(downloadedPackages, this.eventStream);
+            installationStage = 'downloadAndInstallPackages';
+            await this.packageManager.DownloadAndInstallPackages(packages, this.vscode, this.platformInfo, this.eventStream);
 
             this.eventStream.post(new InstallationSuccess());
         }

diff --git a/src/packageManager/PackageDownloader.ts b/src/packageManager/PackageDownloader.ts
@@ -0,0 +1,145 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as tmp from 'tmp';
+import * as https from 'https';
+import * as util from '../common';
+import * as fs from 'fs';
+import { EventStream } from "../EventStream";
+import { DownloadSuccess, DownloadStart, DownloadFallBack, DownloadFailure, DownloadProgress, DownloadSizeObtained } from "../omnisharp/loggingEvents";
+import { Package, PackageError } from "./packages";
+import { parse as parseUrl } from 'url';
+import { getProxyAgent } from './proxy';
+import { vscode } from '../vscodeAdapter';
+
+export class PackageDownloader {
+    constructor() {
+        // Ensure our temp files get cleaned up in case of error.
+        tmp.setGracefulCleanup();
+    }
+
+    public async DownloadPackage(description: string, url: string, fallbackUrl: string, vscode: vscode, eventStream: EventStream) {
+        const config = vscode.workspace.getConfiguration();
+        const proxy = config.get<string>('http.proxy');
+        const strictSSL = config.get('http.proxyStrictSSL', true);
+        return downloadPackage(pkg, eventStream, proxy, strictSSL);
+    }
+}
+
+// We dont need this as we are already doing this check in the package manager
+/*async function maybeDownloadPackage(pkg: Package, eventStream: EventStream, proxy: string, strictSSL: boolean): Promise<Package> {
+    let exists = await doesPackageTestPathExist(pkg);
+    if (!exists) {
+        return downloadPackage(pkg, eventStream, proxy, strictSSL);
+    } else {
+        eventStream.post(new DownloadSuccess(`Skipping package '${pkg.description}' (already downloaded).`));
+        return pkg;
+    }
+}*/
+
+async function downloadPackage(description: string, url: string, fallbackUrl: string, eventStream: EventStream, proxy: string, strictSSL: boolean): Promise<Package> {
+    eventStream.post(new DownloadStart(description));
+    const tmpResult = await new Promise<tmp.SynchrounousResult>((resolve, reject) => {
+        tmp.file({ prefix: 'package-' }, (err, path, fd, cleanupCallback) => {
+            if (err) {
+                return reject(new PackageError('Error from tmp.file', description, err));
+            }
+
+            resolve(<tmp.SynchrounousResult>{ name: path, fd: fd, removeCallback: cleanupCallback });
+        });
+    });
+
+    try {
+        await downloadFile(tmpResult, description,url, eventStream, proxy, strictSSL);
+        eventStream.post(new DownloadSuccess(` Done!`));
+    }
+    catch (primaryUrlError) {
+        // If the package has a fallback Url, and downloading from the primary Url failed, try again from 
+        // the fallback. This is used for debugger packages as some users have had issues downloading from
+        // the CDN link
+        if (fallbackUrl) {
+            eventStream.post(new DownloadFallBack(fallbackUrl));
+            try {
+                await downloadFile(tmpFile, description, fallbackUrl, eventStream, proxy, strictSSL);
+                eventStream.post(new DownloadSuccess(' Done!'));
+            }
+            catch (fallbackUrlError) {
+                throw primaryUrlError;
+            }
+        }
+        else {
+            throw primaryUrlError;
+        }
+    }
+}
+
+async function downloadFile(tmpFile: tmp.SynchrounousResult, description: string, urlString: string, eventStream: EventStream, proxy: string, strictSSL: boolean): Promise<void> {
+    const url = parseUrl(urlString);
+
+    const options: https.RequestOptions = {
+        host: url.hostname,
+        path: url.path,
+        agent: getProxyAgent(url, proxy, strictSSL),
+        port: url.port,
+        rejectUnauthorized: util.isBoolean(strictSSL) ? strictSSL : true
+    };
+
+    return new Promise<void>((resolve, reject) => {
+        if (!tmpFile || tmpFile.fd == 0) {
+            return reject(new PackageError("Temporary package file unavailable", pkg));
+        }
+
+        let request = https.request(options, response => {
+            if (response.statusCode === 301 || response.statusCode === 302) {
+                // Redirect - download from new location
+                return resolve(downloadFile(tmpFile, description, response.headers.location, eventStream, proxy, strictSSL));
+            }
+
+            if (response.statusCode != 200) {
+                // Download failed - print error message
+                eventStream.post(new DownloadFailure(`failed (error code '${response.statusCode}')`));
+                return reject(new PackageError(response.statusCode.toString(), description));
+            }
+
+            // Downloading - hook up events
+            let packageSize = parseInt(response.headers['content-length'], 10);
+            let downloadedBytes = 0;
+            let downloadPercentage = 0;
+            let tmpFile = fs.createWriteStream(null, { fd: tmpFile.fd });
+
+            eventStream.post(new DownloadSizeObtained(packageSize));
+
+            response.on('data', data => {
+                downloadedBytes += data.length;
+
+                // Update status bar item with percentage
+                let newPercentage = Math.ceil(100 * (downloadedBytes / packageSize));
+                if (newPercentage !== downloadPercentage) {
+                    downloadPercentage = newPercentage;
+                    eventStream.post(new DownloadProgress(downloadPercentage, description));
+                }
+            });
+
+            response.on('end', () => {
+                resolve();
+            });
+
+            response.on('error', err => {
+                reject(new PackageError(`Reponse error: ${err.message || 'NONE'}`, description, err));
+            });
+
+            // Begin piping data from the response to the package file
+            response.pipe(tmpFile, { end: false });
+        });
+
+        request.on('error', err => {
+            console.log(err);
+            reject(new PackageError(`Request error: ${err.message || 'NONE'}`, description, err));
+        });
+
+        // Execute the request
+        request.end();
+    });
+}
diff --git a/src/packageManager/PackageInstaller.ts b/src/packageManager/PackageInstaller.ts
@@ -0,0 +1,109 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { Package, PackageError, getPackageTestPath, getBaseInstallPath } from "./packages";
+import { EventStream } from "../EventStream";
+import { InstallationProgress } from "../omnisharp/loggingEvents";
+import * as fs from 'fs';
+import * as mkdirp from 'mkdirp';
+import * as yauzl from 'yauzl';
+import * as path from 'path';
+
+export class PackageInstaller {
+    constructor() { }
+
+    public InstallPackage(pkg: Package, eventStream: EventStream): Promise<void> {
+        const installationStage = 'installPackages';
+        if (!pkg.tmpFile) {
+            // Download of this package was skipped, so there is nothing to install
+            return Promise.resolve();
+        }
+
+        resolvePackageBinaries(pkg);
+        eventStream.post(new InstallationProgress(installationStage, pkg.description));
+
+        return new Promise<void>((resolve, baseReject) => {
+            const reject = (err: any) => {
+                // If anything goes wrong with unzip, make sure we delete the test path (if there is one)
+                // so we will retry again later
+                const testPath = getPackageTestPath(pkg);
+                if (testPath) {
+                    fs.unlink(testPath, unlinkErr => {
+                        baseReject(err);
+                    });
+                } else {
+                    baseReject(err);
+                }
+            };
+
+            if (pkg.tmpFile.fd == 0) {
+                return reject(new PackageError('Downloaded file unavailable', pkg));
+            }
+
+            yauzl.fromFd(pkg.tmpFile.fd, { lazyEntries: true }, (err, zipFile) => {
+                if (err) {
+                    return reject(new PackageError('Immediate zip file error', pkg, err));
+                }
+
+                zipFile.readEntry();
+
+                zipFile.on('entry', (entry: yauzl.Entry) => {
+                    let absoluteEntryPath = path.resolve(getBaseInstallPath(pkg), entry.fileName);
+
+                    if (entry.fileName.endsWith('/')) {
+                        // Directory - create it
+                        mkdirp(absoluteEntryPath, { mode: 0o775 }, err => {
+                            if (err) {
+                                return reject(new PackageError('Error creating directory for zip directory entry:' + err.code || '', pkg, err));
+                            }
+
+                            zipFile.readEntry();
+                        });
+                    }
+                    else {
+                        // File - extract it
+                        zipFile.openReadStream(entry, (err, readStream) => {
+                            if (err) {
+                                return reject(new PackageError('Error reading zip stream', pkg, err));
+                            }
+
+                            mkdirp(path.dirname(absoluteEntryPath), { mode: 0o775 }, err => {
+                                if (err) {
+                                    return reject(new PackageError('Error creating directory for zip file entry', pkg, err));
+                                }
+
+                                // Make sure executable files have correct permissions when extracted
+                                let fileMode = pkg.binaries && pkg.binaries.indexOf(absoluteEntryPath) !== -1
+                                    ? 0o755
+                                    : 0o664;
+
+                                readStream.pipe(fs.createWriteStream(absoluteEntryPath, { mode: fileMode }));
+                                readStream.on('end', () => zipFile.readEntry());
+                            });
+                        });
+                    }
+                });
+
+                zipFile.on('end', () => {
+                    resolve();
+                });
+
+                zipFile.on('error', err => {
+                    reject(new PackageError('Zip File Error:' + err.code || '', pkg, err));
+                });
+            });
+        }).then(() => {
+            // Clean up temp file
+            pkg.tmpFile.removeCallback();
+        });
+    }
+}
+
+function resolvePackageBinaries(pkg: Package) {
+    // Convert relative binary paths to absolute
+    if (pkg.binaries) {
+        pkg.binaries = pkg.binaries.map(value => path.resolve(getBaseInstallPath(pkg), value));
+    }
+}