From 2ea2c4a663d0acb229c33f6a79a4ec3c172fe053 Mon Sep 17 00:00:00 2001 From: roogue Date: Thu, 5 Jan 2023 11:34:40 +0800 Subject: [PATCH 01/51] Update settings.json --- .vscode/settings.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.vscode/settings.json b/.vscode/settings.json index 608559a..ed6510a 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -7,6 +7,7 @@ "domcontentloaded", "Ocdl", "Osdb", + "taiko", "undici" ] } \ No newline at end of file From 5e10851ad5ab50ebebfc9f399ed0bf4ce41f0074 Mon Sep 17 00:00:00 2001 From: roogue Date: Thu, 5 Jan 2023 11:36:01 +0800 Subject: [PATCH 02/51] Added structures to arrange collections --- src/core/Monitor.ts | 140 +++++++++++++++++++++++++++++++++++++++ src/struct/BeatMap.ts | 22 ++++++ src/struct/BeatMapSet.ts | 36 ++++++++++ src/struct/Collection.ts | 88 ++++++++++++++++++++++++ 4 files changed, 286 insertions(+) create mode 100644 src/core/Monitor.ts create mode 100644 src/struct/BeatMap.ts create mode 100644 src/struct/BeatMapSet.ts create mode 100644 src/struct/Collection.ts diff --git a/src/core/Monitor.ts b/src/core/Monitor.ts new file mode 100644 index 0000000..a06ce9e --- /dev/null +++ b/src/core/Monitor.ts @@ -0,0 +1,140 @@ +import chalk from "chalk"; +import { log, clear } from "console"; +import { config } from "../config"; +import type { Collection } from "../struct/Collection"; +import { Message, Msg } from "../struct/Message"; +import OcdlError from "../struct/OcdlError"; + +interface Condition { + mode: string; + retry_input: boolean; + retry_mode: boolean; + fetched_collection: number; + downloaded_beatmapset: number; +} + +export default class Monitor { + private readonly version: string; + private progress = 0; + private prompt = require("prompt-sync")({ sigint: true }); + private readonly task: Record void>; + + readonly collection: Collection; + + readonly condition: Condition; + + constructor(collection: Collection) { + this.collection = collection; + + this.condition = { + mode: config.mode.toString(), + retry_input: false, + retry_mode: false, + fetched_collection: 0, + downloaded_beatmapset: 0, + }; + + this.version = (require("../../package.json")?.version ?? + "Unknown") as string; // Get current version from package.json + + this.task = { + 0: () => {}, // Empty function + 1: this.p_input_id.bind(this), // Input id + 2: this.p_input_mode.bind(this), // Input mode + 3: this.p_fetch_collection.bind(this), // Fetch collection + 4: this.p_create_folder.bind(this), // Fetch collection v2 + 5: this.p_generate_osdb.bind(this), // Generate osdb + 6: this.p_download.bind(this), // Download beatmapset + }; + } + + update(): void { + if (1 != 1) clear(); + // Header + log(chalk.yellow(`osu-collector-dl v${this.version}`)); + log( + chalk.green( + `Collection: ${this.collection.id} - ${this.collection.name} | Mode: ${this.condition.mode}` + ) + ); + // Display progress according to current task + try { + this.task[this.progress](); + } catch (e) { + throw new OcdlError("MESSAGE_GENERATOR_FAILED", e); + } + } + + freeze(message: string, isErrored: boolean = false): void { + // Red color if errored, green if not + log(isErrored ? chalk.red(message) : chalk.greenBright(message)); + + // Freeze the console with prompt + this.prompt(`Press "Enter" to ${isErrored ? "exit" : "continue"}.`); + + if (isErrored) process.exit(1); + } + + awaitInput(message: string, value?: any): string { + return this.prompt(message + " ", value); // Add space + } + + // Keep progress on track + next(): void { + this.progress++; + } + + setCondition(new_condition: Record): void { + Object.assign(this.condition, new_condition); + } + + // Task 1 + private p_input_id(): void { + if (this.condition.retry_input) { + log(chalk.red(new Message(Msg.INPUT_ID_ERR).toString())); + } + } + + // Task 2 + private p_input_mode(): void { + if (this.condition.retry_mode) { + log(chalk.red(new Message(Msg.INPUT_MODE_ERR).toString())); + } + } + + // Task 3 + private p_fetch_collection(): void { + const beatmaps_length = this.collection.beatMapCount.toString(); + + log( + new Message(Msg.FETCH_DATA, { + amount: this.condition.fetched_collection.toString(), + length: beatmaps_length, + }).toString() + ); + } + + // Task 4 + private p_create_folder(): void { + log( + new Message(Msg.CREATE_FOLDER, { name: this.collection.name }).toString() + ); + } + + // Task 5 + private p_generate_osdb(): void { + log( + new Message(Msg.GENERATE_OSDB, { name: this.collection.name }).toString() + ); + } + + // Task 5 + private p_download(): void { + log( + new Message(Msg.GENERATE_OSDB, { + amount: this.condition.downloaded_beatmapset.toString(), + total: this.collection.beatMapSets.size.toString(), + }).toString() + ); + } +} diff --git a/src/struct/BeatMap.ts b/src/struct/BeatMap.ts new file mode 100644 index 0000000..13b5f84 --- /dev/null +++ b/src/struct/BeatMap.ts @@ -0,0 +1,22 @@ +import Util from "../util"; +import OcdlError from "./OcdlError"; + +export class BeatMap { + // compulsory + id: number; + checksum: string; + + // nullable + version?: string; + mode?: number; + difficulty_rating?: number; + + constructor(jsonData: Record) { + const { id, checksum } = jsonData; + const und = Util.checkUndefined({ id, checksum }); + if (und) throw new OcdlError("CORRUPTED_RESPONSE", `${und} is required`); + + this.id = id; + this.checksum = checksum; + } +} diff --git a/src/struct/BeatMapSet.ts b/src/struct/BeatMapSet.ts new file mode 100644 index 0000000..b60a72f --- /dev/null +++ b/src/struct/BeatMapSet.ts @@ -0,0 +1,36 @@ +import Util from "../util"; +import { BeatMap } from "./BeatMap"; +import OcdlError from "./OcdlError"; + +export class BeatMapSet { + // compulsory + id: number; + beatMaps: Map; + + // nullable + title?: string + artist?: string + + constructor(jsonData: Record) { + const { id, beatmaps } = jsonData; + const und = Util.checkUndefined({ id, beatmaps }); + if (und) throw new OcdlError("CORRUPTED_RESPONSE", `${und} is required`); + + this.id = id; + this.beatMaps = this._resolveBeatMaps(beatmaps); + } + + private _resolveBeatMaps(array: Record[]) { + const resolvedData = new Map(); + const unresolvedData = array; + for (let i = 0; i < unresolvedData.length; i++) { + try { + const map = new BeatMap(unresolvedData[i]); + resolvedData.set(map.id, map); + } catch (e) { + throw new OcdlError("CORRUPTED_RESPONSE", e); + } + } + return resolvedData; + } +} diff --git a/src/struct/Collection.ts b/src/struct/Collection.ts new file mode 100644 index 0000000..08003b5 --- /dev/null +++ b/src/struct/Collection.ts @@ -0,0 +1,88 @@ +import OcdlError from "./OcdlError"; +import { BeatMapSet } from "./BeatMapSet"; +import Util from "../util"; +import { ModeByte } from "../types"; + +export class Collection { + beatMapSets: Map = new Map(); + beatMapCount: number = 0; + id: number = 0; + name: string = "Unknown"; + uploader: { + username: string; + } = { + username: "Unknown", + }; + + constructor() {} + + resolveData(jsonData: Record = {}) { + const { id, name, uploader, beatmapsets, beatmapCount } = jsonData; + const und = Util.checkUndefined({ + id, + name, + uploader, + beatmapsets, + beatmapCount, + }); + if (und) throw new OcdlError("CORRUPTED_RESPONSE", `${und} is required`); + + this.id = id; + this.name = name; + this.uploader = uploader; + this.beatMapSets = this._resolveBeatMapSets(beatmapsets); + this.beatMapCount = beatmapCount; + } + + resolveFullData(array: Record[]): void { + const unresolvedData = array; + if (!unresolvedData.length) + throw new OcdlError("CORRUPTED_RESPONSE", "No beatmap found"); + + for (let i = 0; i < unresolvedData.length; i++) { + const { id, mode, difficulty_rating, version, beatmapset } = array[i]; + const und = Util.checkUndefined({ + id, + mode, + difficulty_rating, + version, + beatmapset, + }); + if (und) throw new OcdlError("CORRUPTED_RESPONSE", `${und} is required`); + + const beatMapSet = this.beatMapSets.get(beatmapset.id); + if (!beatMapSet) continue; + + const { title, artist } = beatmapset; + + beatMapSet.title = title; + beatMapSet.artist = artist; + + const beatMap = beatMapSet.beatMaps.get(id); + if (!beatMap) continue; + + beatMap.difficulty_rating = difficulty_rating; + beatMap.mode = +ModeByte[mode]; + beatMap.version = version; + } + } + + private _resolveBeatMapSets( + array: Record[] + ): Map { + const resolvedData = new Map(); + const unresolvedData = array; + if (!unresolvedData.length) + throw new OcdlError("CORRUPTED_RESPONSE", "No beatmapset found"); + + for (let i = 0; i < unresolvedData.length; i++) { + try { + const set = new BeatMapSet(unresolvedData[i]); + resolvedData.set(set.id, set); + } catch (e) { + throw new OcdlError("CORRUPTED_RESPONSE", e); + } + } + return resolvedData; + } +} From 558f7ac3f08a67ef0e8e939f6fb7b589defc9137 Mon Sep 17 00:00:00 2001 From: roogue Date: Thu, 5 Jan 2023 11:36:21 +0800 Subject: [PATCH 03/51] Remove stayalivelog and replace with freeze --- src/core/Logger.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/core/Logger.ts b/src/core/Logger.ts index 7060aed..27d5766 100644 --- a/src/core/Logger.ts +++ b/src/core/Logger.ts @@ -6,12 +6,6 @@ export default class Logger { static readonly errorLogPath = "./ocdl-error.log"; static readonly missingLogPath = "./ocdl-missing.log"; - static async stayAliveLog(message: string) { - console.error(message); - await new Promise((res) => setTimeout(res, 5000)); // Sleep for 5 seconds before closing console - throw new Error(); - } - static generateErrorLog(error: OcdlError): boolean { try { if (!Logger.checkIfErrorLogFileExists()) { From 396957657f7c44791d8e1fd3448b87778b737219 Mon Sep 17 00:00:00 2001 From: roogue Date: Thu, 5 Jan 2023 11:36:39 +0800 Subject: [PATCH 04/51] Added Message enums --- src/struct/Message.ts | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 src/struct/Message.ts diff --git a/src/struct/Message.ts b/src/struct/Message.ts new file mode 100644 index 0000000..55d4765 --- /dev/null +++ b/src/struct/Message.ts @@ -0,0 +1,36 @@ +export class Message { + private message: Msg; + private variable: Record; + + constructor(message: Msg, variable?: Record) { + this.message = message; + this.variable = variable ?? {}; + } + + toString() { + // Replace value if variable is provided + let msg: string = this.message; + for (const [key, value] of Object.entries(this.variable)) { + const regex = new RegExp(`{{${key}}}`, "g"); + msg = msg.replace(regex, value); + } + return msg; + } +} + +export enum Msg { + NO_CONNECTION = "This script only runs with presence of internet connection.", + + INPUT_ID = "Enter the collection ID you want to download:", + INPUT_ID_ERR = "ID should be a number, Ex: '44' (without the quote)", + INPUT_MODE = "Generate .osdb file? (y/n) (Default: {{mode}}):", + INPUT_MODE_ERR = "Invalid mode, please type 'y' or 'n' (without the quote)", + + FETCH_DATA = "Fetched data of [ {{amount}}/{{total}} ] beatmaps...", + + CREATE_FOLDER = "Creating folder {{name}}...", + + GENERATE_OSDB = "Generating {{name}}.osdb file", + + DOWNLOAD_SONG = "Downloading [ {{amount}}/{{total}} ] beatmap set...", +} From a4b0b28ae0e6fac8f90595581542640bb76b2c02 Mon Sep 17 00:00:00 2001 From: roogue Date: Thu, 5 Jan 2023 11:37:07 +0800 Subject: [PATCH 05/51] Updated packages to latest version --- package.json | 5 +-- tsconfig.json | 4 +-- yarn.lock | 90 +++++++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 85 insertions(+), 14 deletions(-) diff --git a/package.json b/package.json index 217638c..4066a31 100644 --- a/package.json +++ b/package.json @@ -15,10 +15,11 @@ }, "license": "MIT", "dependencies": { + "chalk": "4.1.2", "csbinary": "^2.1.4", "prompt-sync": "^4.2.0", - "tslib": "^2.4.0", - "undici": "^5.8.0" + "tslib": "^2.4.1", + "undici": "^5.14.0" }, "devDependencies": { "@types/node": "^18.0.6" diff --git a/tsconfig.json b/tsconfig.json index 89c9d51..11e6d20 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -30,6 +30,6 @@ "forceConsistentCasingInFileNames": true, "downlevelIteration": true, "skipLibCheck": true, - "resolveJsonModule": true, + "resolveJsonModule": true } -} \ No newline at end of file +} diff --git a/yarn.lock b/yarn.lock index 78e3393..1556d60 100644 --- a/yarn.lock +++ b/yarn.lock @@ -97,6 +97,15 @@ __metadata: languageName: node linkType: hard +"ansi-styles@npm:^4.1.0": + version: 4.3.0 + resolution: "ansi-styles@npm:4.3.0" + dependencies: + color-convert: ^2.0.1 + checksum: 513b44c3b2105dd14cc42a19271e80f386466c4be574bccf60b627432f9198571ebf4ab1e4c3ba17347658f4ee1711c163d574248c0c1cdc2d5917a0ad582ec4 + languageName: node + linkType: hard + "aproba@npm:^1.0.3 || ^2.0.0": version: 2.0.0 resolution: "aproba@npm:2.0.0" @@ -140,6 +149,15 @@ __metadata: languageName: node linkType: hard +"busboy@npm:^1.6.0": + version: 1.6.0 + resolution: "busboy@npm:1.6.0" + dependencies: + streamsearch: ^1.1.0 + checksum: 32801e2c0164e12106bf236291a00795c3c4e4b709ae02132883fe8478ba2ae23743b11c5735a0aae8afe65ac4b6ca4568b91f0d9fed1fdbc32ede824a73746e + languageName: node + linkType: hard + "cacache@npm:^16.1.0": version: 16.1.1 resolution: "cacache@npm:16.1.1" @@ -166,6 +184,16 @@ __metadata: languageName: node linkType: hard +"chalk@npm:4.1.2": + version: 4.1.2 + resolution: "chalk@npm:4.1.2" + dependencies: + ansi-styles: ^4.1.0 + supports-color: ^7.1.0 + checksum: fe75c9d5c76a7a98d45495b91b2172fa3b7a09e0cc9370e5c8feb1c567b85c4288e2b3fded7cfdd7359ac28d6b3844feb8b82b8686842e93d23c827c417e83fc + languageName: node + linkType: hard + "chownr@npm:^2.0.0": version: 2.0.0 resolution: "chownr@npm:2.0.0" @@ -180,6 +208,22 @@ __metadata: languageName: node linkType: hard +"color-convert@npm:^2.0.1": + version: 2.0.1 + resolution: "color-convert@npm:2.0.1" + dependencies: + color-name: ~1.1.4 + checksum: 79e6bdb9fd479a205c71d89574fccfb22bd9053bd98c6c4d870d65c132e5e904e6034978e55b43d69fcaa7433af2016ee203ce76eeba9cfa554b373e7f7db336 + languageName: node + linkType: hard + +"color-name@npm:~1.1.4": + version: 1.1.4 + resolution: "color-name@npm:1.1.4" + checksum: b0445859521eb4021cd0fb0cc1a75cecf67fceecae89b63f62b201cca8d345baf8b952c966862a9d9a2632987d4f6581f0ec8d957dfacece86f0a7919316f610 + languageName: node + linkType: hard + "color-support@npm:^1.1.3": version: 1.1.3 resolution: "color-support@npm:1.1.3" @@ -337,6 +381,13 @@ __metadata: languageName: node linkType: hard +"has-flag@npm:^4.0.0": + version: 4.0.0 + resolution: "has-flag@npm:4.0.0" + checksum: 261a1357037ead75e338156b1f9452c016a37dcd3283a972a30d9e4a87441ba372c8b81f818cd0fbcd9c0354b4ae7e18b9e1afa1971164aef6d18c2b6095a8ad + languageName: node + linkType: hard + "has-unicode@npm:^2.0.1": version: 2.0.1 resolution: "has-unicode@npm:2.0.1" @@ -700,10 +751,11 @@ __metadata: resolution: "osu-collector-dl@workspace:." dependencies: "@types/node": ^18.0.6 + chalk: 4.1.2 csbinary: ^2.1.4 prompt-sync: ^4.2.0 - tslib: ^2.4.0 - undici: ^5.8.0 + tslib: ^2.4.1 + undici: ^5.14.0 languageName: unknown linkType: soft @@ -854,6 +906,13 @@ __metadata: languageName: node linkType: hard +"streamsearch@npm:^1.1.0": + version: 1.1.0 + resolution: "streamsearch@npm:1.1.0" + checksum: 1cce16cea8405d7a233d32ca5e00a00169cc0e19fbc02aa839959985f267335d435c07f96e5e0edd0eadc6d39c98d5435fb5bbbdefc62c41834eadc5622ad942 + languageName: node + linkType: hard + "string-width@npm:^1.0.2 || 2 || 3 || 4, string-width@npm:^4.2.3": version: 4.2.3 resolution: "string-width@npm:4.2.3" @@ -892,6 +951,15 @@ __metadata: languageName: node linkType: hard +"supports-color@npm:^7.1.0": + version: 7.2.0 + resolution: "supports-color@npm:7.2.0" + dependencies: + has-flag: ^4.0.0 + checksum: 3dda818de06ebbe5b9653e07842d9479f3555ebc77e9a0280caf5a14fb877ffee9ed57007c3b78f5a6324b8dbeec648d9e97a24e2ed9fdb81ddc69ea07100f4a + languageName: node + linkType: hard + "tar@npm:^6.1.11, tar@npm:^6.1.2": version: 6.1.11 resolution: "tar@npm:6.1.11" @@ -906,17 +974,19 @@ __metadata: languageName: node linkType: hard -"tslib@npm:^2.4.0": - version: 2.4.0 - resolution: "tslib@npm:2.4.0" - checksum: 8c4aa6a3c5a754bf76aefc38026134180c053b7bd2f81338cb5e5ebf96fefa0f417bff221592bf801077f5bf990562f6264fecbc42cd3309b33872cb6fc3b113 +"tslib@npm:^2.4.1": + version: 2.4.1 + resolution: "tslib@npm:2.4.1" + checksum: 19480d6e0313292bd6505d4efe096a6b31c70e21cf08b5febf4da62e95c265c8f571f7b36fcc3d1a17e068032f59c269fab3459d6cd3ed6949eafecf64315fca languageName: node linkType: hard -"undici@npm:^5.8.0": - version: 5.8.0 - resolution: "undici@npm:5.8.0" - checksum: 7b486ad064da00628d3906e140b86223023cd3494c811da8d7aa1375c2392fe6a6ac421af236c056fd3d3136bba3a91b99e0505dde071dd946070946eb0718b8 +"undici@npm:^5.14.0": + version: 5.14.0 + resolution: "undici@npm:5.14.0" + dependencies: + busboy: ^1.6.0 + checksum: 7a076e44d84b25844b4eb657034437b8b9bb91f17d347de474fdea1d4263ce7ae9406db79cd30de5642519277b4893f43073258bcc8fed420b295da3fdd11b26 languageName: node linkType: hard From 33356f150676e73b1a6f9f16ee73da373d133559 Mon Sep 17 00:00:00 2001 From: roogue Date: Thu, 5 Jan 2023 11:37:50 +0800 Subject: [PATCH 06/51] Fix bug in extraction of filename --- src/core/DownloadManager.ts | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/src/core/DownloadManager.ts b/src/core/DownloadManager.ts index 9ed68af..af3ed7e 100644 --- a/src/core/DownloadManager.ts +++ b/src/core/DownloadManager.ts @@ -1,31 +1,33 @@ import { createWriteStream, existsSync } from "fs"; -import type Config from "../struct/Config"; import { Response, fetch } from "undici"; import _path from "path"; import Logger from "./Logger"; import OcdlError from "../struct/OcdlError"; -import type { Collection } from "../types"; import Util from "../util"; +import type Monitor from "./Monitor"; +import { config } from "../config"; +import EventEmitter from "events"; -export class DownloadManager { +export class DownloadManager extends EventEmitter { + monitor: Monitor; path: string; parallel: boolean; impulseRate: number; osuMirrorUrl: string; altOsuMirrorUrl: string; - collection: Collection; - constructor(config: Config, collection: Collection) { - this.path = _path.join(config.directory, collection.name); + constructor(monitor: Monitor) { + super(); + this.monitor = monitor; + this.path = _path.join(config.directory, monitor.collection.name); this.parallel = config.parallel; this.impulseRate = config.dl_impulse_rate; this.osuMirrorUrl = config.osuMirrorApiUrl; this.altOsuMirrorUrl = config.altOsuMirrorUrl; - this.collection = collection; } public async bulk_download(): Promise { - const ids = this.collection.beatmapsets.map((beatmapSet) => beatmapSet.id); + const ids = Array.from(this.monitor.collection.beatMapSets.keys()); if (this.parallel) { // Impulsive download if url length is more then this.impulseRate @@ -92,9 +94,7 @@ export class DownloadManager { console.log("Downloaded: " + url); } catch (e) { - Logger.generateErrorLog(new OcdlError("REQUEST_DOWNLOAD_FAILED", e)); - } finally { - return; + throw new OcdlError("DOWNLOAD_FAILED", e); } } @@ -105,7 +105,7 @@ export class DownloadManager { let fileName = "Untitled.osz"; // Default file name // Extract filename from content-disposition header. if (contentDisposition) { - const result = /filename="(.+)"/g.exec(contentDisposition); + const result = /filename=([^;]+)/g.exec(contentDisposition); if (result) { try { @@ -114,9 +114,7 @@ export class DownloadManager { fileName = decoded; } catch (e) { - Logger.generateErrorLog( - new OcdlError("FILE_NAME_EXTRACTION_FAILED", e) - ); + throw new OcdlError("FILE_NAME_EXTRACTION_FAILED", e); } } } From 1812405a92aa11c272660bc7e940ddead5e0e188 Mon Sep 17 00:00:00 2001 From: roogue Date: Thu, 5 Jan 2023 11:38:35 +0800 Subject: [PATCH 07/51] Restructure and Factorization --- src/core/Main.ts | 175 +++++++++++++++++++++----------------- src/core/OsdbGenerator.ts | 70 +++++++-------- src/index.ts | 87 +++++++++++-------- src/util.ts | 9 +- 4 files changed, 194 insertions(+), 147 deletions(-) diff --git a/src/core/Main.ts b/src/core/Main.ts index 3209cec..9dfe669 100644 --- a/src/core/Main.ts +++ b/src/core/Main.ts @@ -1,122 +1,145 @@ import { DownloadManager } from "./DownloadManager"; import { request } from "undici"; import type Config from "../struct/Config"; -import Logger from "./Logger"; -import type { BeatMapV2, Collection, BeatMapV2ResData } from "../types"; -import type { ResponseData } from "undici/types/dispatcher"; import OsdbGenerator from "./OsdbGenerator"; import OcdlError from "../struct/OcdlError"; import { existsSync, mkdirSync } from "fs"; import _path from "path"; import Util from "../util"; +import type Monitor from "./Monitor"; +import { config } from "../config"; export default class Main { + monitor: Monitor; + config: Config = config; collectionApiUrl: string; collectionApiUrlV2: string; - config: Config; - constructor(id: number, config: Config) { + constructor(monitor: Monitor) { + this.monitor = monitor; + + const id = monitor.collection.id; // Quick hand api url for faster fetching - this.collectionApiUrl = config.osuCollectorApiUrl + id; + this.collectionApiUrl = this.config.osuCollectorApiUrl + id.toString(); // Api url for full information - this.collectionApiUrlV2 = config.osuCollectorApiUrl + id + "/beatmapsV2"; - - this.config = config; + this.collectionApiUrlV2 = + this.config.osuCollectorApiUrl + id.toString() + "/beatmapsV2"; } async run(): Promise { - // Fetch collection - const apiRes = await this.fetchCollection().catch(() => null); - if (!apiRes || apiRes.statusCode !== 200) - return Logger.stayAliveLog( - `Request collection failed. Status Code: ${apiRes?.statusCode}` - ); - // Map beatmapSet ids - const resData: Collection | null = await apiRes.body - .json() - .catch(() => null); - if (!resData || !resData.beatmapsets?.length) - return Logger.stayAliveLog("No beatmap set found."); + // Fetch brief collection info + const responseData = await this.fetchCollection(); + if (responseData instanceof OcdlError) throw responseData; + this.monitor.collection.resolveData(responseData); - // Create folder - try { - resData.name = Util.replaceForbiddenChars(resData.name); - const path = _path.join(this.config.directory, resData.name); - if (!existsSync(path)) mkdirSync(path); - } catch (e) { - Logger.generateErrorLog(new OcdlError("FOLDER_GENERATION_FAILED", e)); - } + // Task 3 + this.monitor.next(); + this.monitor.update(); + // Fetch full data if user wants generate osdb file if (this.config.mode === 2) { - // v2BeatMapInfo Cache - const beatMapV2: BeatMapV2[] = []; + let hasMorePage: boolean = true; + let cursor: number = 0; - let hasMorePage = true; - let cursor = 0; while (hasMorePage) { // Request v2 collection - const v2ApiResponse = await this.fetchCollectionV2(cursor).catch( - () => null - ); + const v2ResponseData = await this.fetchCollection(true, cursor); + if (v2ResponseData instanceof OcdlError) throw v2ResponseData; - if (!v2ApiResponse || v2ApiResponse.statusCode !== 200) - return Logger.stayAliveLog( - `Request collection V2 failed. Status Code: ${v2ApiResponse?.statusCode}` - ); + try { + const { hasMore, nextPageCursor, beatmaps } = v2ResponseData; - const v2ResData: BeatMapV2ResData | null = - await v2ApiResponse.body.json(); - if (!v2ResData) return Logger.stayAliveLog("Bad response."); + const und = Util.checkUndefined({ + hasMore, + nextPageCursor, + beatmaps, + }); + if (und) + throw new OcdlError("CORRUPTED_RESPONSE", `${und} is required`); - try { - const { nextPageCursor, hasMore, beatmaps } = v2ResData; - if (!Util.isBoolean(hasMore)) { - return Logger.stayAliveLog("Bad response."); // As precaution if data is inaccurate - } + // Resolve all required data + this.monitor.collection.resolveFullData(beatmaps); // Set property hasMorePage = hasMore; cursor = nextPageCursor; - // Add beatmap to cache - beatMapV2.push(...beatmaps); - - console.log("Fetched " + beatMapV2.length + " Beatmaps"); + const fetched_collection = + this.monitor.condition.fetched_collection + beatmaps.length; + this.monitor.setCondition({ fetched_collection }); + // this.monitor.update(); } catch (e) { - Logger.generateErrorLog(new OcdlError("REQUEST_DATA_FAILED", e)); + throw new OcdlError("REQUEST_DATA_FAILED", e); } } - // Generate .osdb - console.log("Generating .osdb file..."); - const generator = new OsdbGenerator(this.config, resData, beatMapV2); - await generator.writeOsdb(); - console.log("Generated!"); } + // Task 4 + this.monitor.next(); + this.monitor.update(); + + // Create folder + try { + responseData.name = Util.replaceForbiddenChars(responseData.name); + const path = _path.join(this.config.directory, responseData.name); + if (!existsSync(path)) mkdirSync(path); + } catch (e) { + throw new OcdlError("FOLDER_GENERATION_FAILED", e); + } + + // Task 5 + this.monitor.next(); + this.monitor.update(); + + // Generate .osdb file + if (this.config.mode === 2) { + try { + const generator = new OsdbGenerator(this.monitor); + await generator.writeOsdb(); + } catch (e) { + throw new OcdlError("GENERATE_OSDB_FAILED", e); + } + } + + // Task 6 + this.monitor.next(); + this.monitor.update(); + // Download beatmapSet - console.log("Start Downloading..."); - const downloadManager = new DownloadManager(this.config, resData); - await downloadManager.bulk_download(); + try { + const downloadManager = new DownloadManager(this.monitor); + await downloadManager.bulk_download(); + } catch (e) { + throw e; + } - console.log("Download Finished!"); - const prompt = require("prompt-sync")({ sigint: true }); - prompt("Press Enter to exit."); + this.monitor.freeze("Download finished"); return; } - private async fetchCollection(): Promise { - return await request(this.collectionApiUrl, { method: "GET" }); - } - - private async fetchCollectionV2(cursor: number = 0): Promise { - return await request(this.collectionApiUrlV2, { - method: "GET", - query: { - perPage: 100, - cursor, - }, - }); + private async fetchCollection( + v2: boolean = false, + cursor: number = 0 + ): Promise | OcdlError> { + // Check version of collection + const url = v2 ? this.collectionApiUrlV2 : this.collectionApiUrl; + const query: Record = // Query is needed for V2 collection + v2 + ? { + perPage: 100, + cursor, + } + : {}; + const data = await request(url, { method: "GET", query }) + .then(async (res) => { + if (res.statusCode !== 200) throw `Status code: ${res.statusCode}`; + return (await res.body.json()) as Record; + }) + .catch((e) => new OcdlError("REQUEST_DATA_FAILED", e)); + if (data instanceof OcdlError) throw data; + + return data; } } diff --git a/src/core/OsdbGenerator.ts b/src/core/OsdbGenerator.ts index 50b072a..b3ff7d5 100644 --- a/src/core/OsdbGenerator.ts +++ b/src/core/OsdbGenerator.ts @@ -1,24 +1,21 @@ import { BinaryWriter, File, IFile } from "csbinary"; import { openSync, writeFileSync } from "fs"; -import type Config from "../struct/Config"; -import OcdlError from "../struct/OcdlError"; -import { BeatMapV2, Collection, ModeByte } from "../types"; -import Logger from "./Logger"; +import { config } from "../config"; import _path from "path"; +import type Monitor from "./Monitor"; export default class OsdbGenerator { filePath: string; fileName: string; file: IFile; writer: BinaryWriter; - collection: Collection; - beatMaps: BeatMapV2[]; + monitor: Monitor; - constructor(config: Config, collection: Collection, beatMaps: BeatMapV2[]) { - this.fileName = collection.name + ".osdb"; + constructor(monitor: Monitor) { + this.fileName = monitor.collection.name + ".osdb"; this.filePath = _path.join( config.directory, - collection.name, // Folder name + monitor.collection.name, // Folder name this.fileName ); // Create file @@ -28,8 +25,7 @@ export default class OsdbGenerator { this.writer = new BinaryWriter(this.file); - this.collection = collection; - this.beatMaps = beatMaps; + this.monitor = monitor; } // * Refer https://github.com/Piotrekol/CollectionManager/blob/master/CollectionManagerDll/Modules/FileIO/FileCollections/OsdbCollectionHandler.cs#L89 @@ -42,43 +38,47 @@ export default class OsdbGenerator { this.writer.writeDouble(this.toOADate(new Date())); // Editor - this.writer.writeString(this.collection.uploader.username); + this.writer.writeString(this.monitor.collection.uploader.username); // Num of collections this.writer.writeInt32(1); // Always 1 // Name - this.writer.writeString(this.collection.name); + this.writer.writeString(this.monitor.collection.name); // Beatmap count - this.writer.writeInt32(this.beatMaps.length); + this.writer.writeInt32(this.monitor.collection.beatMapCount); - for (const beatmap of this.beatMaps) { - // beatmapId - this.writer.writeInt32(beatmap.id); + this.monitor.collection.beatMapSets.forEach( + (beatMapSet, beatMapSetId) => { + beatMapSet.beatMaps.forEach((beatmap, beatMapId) => { + // BeatmapId + this.writer.writeInt32(beatMapId); - // beatmapSetId - this.writer.writeInt32(beatmap.beatmapset_id); + // BeatmapSetId + this.writer.writeInt32(beatMapSetId); - // Artist - this.writer.writeString(beatmap.beatmapset.artist); - // title - this.writer.writeString(beatmap.beatmapset.title); - // diffname - this.writer.writeString(beatmap.version); + // Artist + this.writer.writeString(beatMapSet.artist ?? "Unknown"); + // Title + this.writer.writeString(beatMapSet.title ?? "Unknown"); + // Version + this.writer.writeString(beatmap.version ?? "Unknown"); - // Md5 - this.writer.writeString(beatmap.checksum); + // Md5 + this.writer.writeString(beatmap.checksum); - // User comment - this.writer.writeString(""); + // User comment + this.writer.writeString(""); - // Play mode - this.writer.writeByte(ModeByte[beatmap.mode]); + // Play mode + this.writer.writeByte(beatmap.mode ?? 0); - // Mod PP Star - this.writer.writeDouble(beatmap.difficulty_rating); - } + // Mod PP Star + this.writer.writeDouble(beatmap.difficulty_rating ?? 0); + }); + } + ); // Map with hash this.writer.writeInt32(0); // Always 0 @@ -86,7 +86,7 @@ export default class OsdbGenerator { // Footer this.writer.writeString("By Piotrekol"); // Fixed Footer } catch (e) { - Logger.generateErrorLog(new OcdlError("GENERATE_OSDB_FAILED", e)); + throw e; } finally { this.closeWriter(); } diff --git a/src/index.ts b/src/index.ts index a55dc99..9ea3cff 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,62 +2,79 @@ import Main from "./core/Main"; import { config } from "./config"; import Logger from "./core/Logger"; import OcdlError from "./struct/OcdlError"; +import Monitor from "./core/Monitor"; +import { Message, Msg } from "./struct/Message"; +import { Collection } from "./struct/Collection"; -// Prompt user for id and mode -const prompt = require("prompt-sync")({ sigint: true }); - -const getId = (): number | null => { - const id: number = Number(prompt("Please Enter An ID: ")); - if (isNaN(id)) { - console.log("Invalid ID, please try again."); - return null; - } - return id; -}; - -const getMode = (): number | null => { - const mode: string = String( - prompt( - `Generate .osdb file? (y/n) - Default(${ - config.mode === 2 ? "Yes" : "No" - }): ` - ) - ).toLowerCase(); - // Select default mode if user doesn't enter anything - if (!mode) return config.mode; - // Check if user entered valid mode - if (["n", "no", "1"].includes(mode)) return 1; - if (["y", "yes", "ass", "2"].includes(mode)) return 2; - - console.log('Invalid mode, please type "y" or "n".'); - return null; +const isOnline = async (): Promise => { + return !!(await require("dns") + .promises.resolve("google.com") + .catch(() => {})); }; +// Script Starts Here (async () => { - // Get version from package.json - const version = (require("../package.json")?.version ?? "Unknown") as string; - console.log(`=== osu-collector-dl v${version} ===`); + // Initiate monitor + const collection = new Collection(); + const monitor = new Monitor(collection); + monitor.update(); + + // Check if internet connection is presence + const onlineStatus = await isOnline(); + if (!onlineStatus) + return monitor.freeze(new Message(Msg.NO_CONNECTION).toString(), true); let id: number | null = null; let mode: number | null = null; try { + // task 1 + monitor.next(); + // Get id while (id === null) { - id = getId(); + monitor.update(); + + const result = Number( + monitor.awaitInput(new Message(Msg.INPUT_ID).toString(), "none") + ); + isNaN(result) ? (monitor.condition.retry_input = true) : (id = result); // check if result is valid } + monitor.collection.id = id; + + // task 2 + monitor.next(); + // Get mode while (mode === null) { - mode = getMode(); + monitor.update(); + const result = String( + monitor.awaitInput( + new Message(Msg.INPUT_MODE, { + mode: config.mode === 2 ? "Yes" : "No", + }).toString(), + config.mode.toString() + ) + ); + if (["n", "no", "1"].includes(result)) mode = 1; + if (["y", "yes", "ass", "2"].includes(result)) mode = 2; + if (mode === null) monitor.condition.retry_mode = true; } + monitor.setCondition({ mode: mode.toString() }); config.mode = mode; } catch (e) { Logger.generateErrorLog(new OcdlError("GET_USER_INPUT_FAILED", e)); return; } - const main = new Main(id, config); - await main.run(); + const main = new Main(monitor); + + try { + await main.run(); + } catch (e) { + console.error(e); + if (e instanceof OcdlError) return Logger.generateErrorLog(e); + } })(); diff --git a/src/util.ts b/src/util.ts index 540ed0c..d6f6526 100644 --- a/src/util.ts +++ b/src/util.ts @@ -4,7 +4,14 @@ export default class Util { } static replaceForbiddenChars(str: string): string { - const regex = /[ \\\/<>:"\|?*]+/g; + const regex = /[\\\/<>:"\|?*]+/g; return str.replace(regex, ""); } + + static checkUndefined(obj: Record): string | null { + for (const [key, value] of Object.entries(obj)) { + if (value === undefined) return key; + } + return null; + } } From 7ea260c709752186f1aa1ce5a3669a025999e52f Mon Sep 17 00:00:00 2001 From: roogue Date: Thu, 5 Jan 2023 11:38:56 +0800 Subject: [PATCH 08/51] More error message --- src/struct/OcdlError.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/struct/OcdlError.ts b/src/struct/OcdlError.ts index 21e451f..fbbdbdf 100644 --- a/src/struct/OcdlError.ts +++ b/src/struct/OcdlError.ts @@ -1,15 +1,16 @@ export enum ErrorType { "GET_USER_INPUT_FAILED" = "Error occurred while getting user input", "RESOLVE_JSON_FAILED" = "Error occurred while resolving res body to json", - "REQUEST_DOWNLOAD_FAILED" = "Error occurred while requesting download", + "DOWNLOAD_FAILED" = "Error occurred while downloading beatmapset", "GENERATE_OSDB_FAILED" = "Error occurred while generating .osdb", "REQUEST_DATA_FAILED" = "Error occurred while requesting data", "FOLDER_GENERATION_FAILED" = "Error occurred while generating folder", "FILE_NAME_EXTRACTION_FAILED" = "Error occurred while extracting file name", + "MESSAGE_GENERATOR_FAILED" = "Error occurred while updating monitor", + "CORRUPTED_RESPONSE" = "The api response is corrupted", } const getMessage = (type: keyof typeof ErrorType, error: any): string => { - console.log(error); return `${new Date()} | [OcdlError]: ${type} - ${ErrorType[type]}\n${error}`; }; From 1a143bd8155fb440c38dd043d03ef335c90cf2ef Mon Sep 17 00:00:00 2001 From: roogue Date: Sat, 7 Jan 2023 12:54:36 +0800 Subject: [PATCH 09/51] Update config.json.example --- config.json.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.json.example b/config.json.example index 4c57240..bf43117 100644 --- a/config.json.example +++ b/config.json.example @@ -1,6 +1,6 @@ { "parallel": true, - "dl_impulse_rate": 5, + "concurrency": 5, "directory": "", "mode": 1 } From 6e95515cde854f66f3196676346dc9333ce1d330 Mon Sep 17 00:00:00 2001 From: roogue Date: Sat, 7 Jan 2023 12:54:40 +0800 Subject: [PATCH 10/51] Update README.md --- README.md | 37 ++++++++++++++----------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 39b45b7..51c44ae 100644 --- a/README.md +++ b/README.md @@ -1,49 +1,40 @@ # osu-collector-dl -A Script that automatizes the downloads of beatmap set in Osu!Collector. +A script that automates the downloading of beatmap sets in Osu!Collector. ## Installation -- Download from the latest release, then extract the compressed file. +1. Download the latest release from the releases page. +2. Extract the compressed file ## Usage -Run the `osu-collector-dl.exe` from the downloaded folder. +1. Run the `osu-collector-dl.exe` file from the downloaded folder. +2. A prompt window will appear. You will need to enter the ID of the collection you want to download. To find the ID, look at the end of the Osu!Collector collection URL. For example, if the URL is https://osucollector.com/collections/44, you would enter 44 as the ID. -After a prompt window pops up, an ID is required. You have to insert the numbers that appear at the end of the Osu!Collector collection url. -For an example: Insert "44" if you want to download from this link: https://osucollector.com/collections/44 +## Configuration -## Config +You can customize various settings that affect the download speed, location of the downloaded beatmaps, and working mode of the script by editing the `config.json` file. To do this, right-click on the file and select "Open with" from the context menu. Choose a text editor (such as Notepad) to open the file and make the desired changes. -You can customize some settings which affect in download speed, location of the downloaded osu beatmaps and working mode. - -Below is the data stored in `config.json`, you can modify it to your flavor. \ -(Tips: Right click and open with notepad to edit .json file.) +Below is the data stored in config.json, along with explanations of each setting: ```json { "parallel": true, - "dl_impulse_rate": 10, + "concurrency": 10, "directory": "", "mode": 1 } ``` -#### Explanation: - -[ parallel: true | false ] - Whether or not downloads should be done in parallel. \ --> If parallel is true, multiple downloads will process at the same time. If false, the program will download only one beatmap set at a time. +- `parallel`: Set to `true` if you want to download multiple beatmap sets at the same time, or `false` if you want to download only one beatmap set at a time. -[ dl_impulse_rate: number ] - How many downloads should be requested at a time. \ --> Warning: this setting is recommended to be set as low as around 5 to prevent unwanted abuse to osu!mirror API. You could get IP banned or rate limited if you're abusing their API (I guess :v but just don't). +- `concurrency`: The number of downloads to request at a time. It is recommended to set this to a low number (such as 5) to prevent abuse of the osu!mirror API and potential IP bans or rate limits. -[ directory: string ] - Path to your download folder. \ --> Remember the double quotes!! If none was provided, the current working directory would be used. +- `directory`: The path to the folder where you want to save the downloaded beatmaps. If no value is provided, the current working directory will be used. Remember to include double quotes around the path! -[ mode: 1 | 2 ] - Which mode should the program works on. \ --> When mode 1 is selected, only download of the beatmap sets will be progressed. If mode 2 is selected, the download will still be progressed but with an additional generation of .osdb file. \ -Note: You can still choose a mode on the terminal. +- `mode`: The mode in which the program should operate. Set to `1` to only download the beatmap sets, or `2` to also generate a .osdb file during the download process. You can also specify the mode at the terminal. ## License -[MIT](https://choosealicense.com/licenses/mit/) +This project is licensed under the MIT License. See the [LICENSE](https://choosealicense.com/licenses/mit/) file for details. \ No newline at end of file From 5c6e91f398c275c6dbf97e1803ed7d24913cfc8d Mon Sep 17 00:00:00 2001 From: roogue Date: Sat, 7 Jan 2023 12:54:58 +0800 Subject: [PATCH 11/51] Rework Download Manager --- src/core/DownloadManager.ts | 175 ++++++++++++++++++------------------ 1 file changed, 90 insertions(+), 85 deletions(-) diff --git a/src/core/DownloadManager.ts b/src/core/DownloadManager.ts index af3ed7e..ec5eeda 100644 --- a/src/core/DownloadManager.ts +++ b/src/core/DownloadManager.ts @@ -1,100 +1,110 @@ import { createWriteStream, existsSync } from "fs"; import { Response, fetch } from "undici"; import _path from "path"; -import Logger from "./Logger"; import OcdlError from "../struct/OcdlError"; import Util from "../util"; +import EventEmitter from "events"; import type Monitor from "./Monitor"; import { config } from "../config"; -import EventEmitter from "events"; +import type { BeatMapSet } from "../struct/BeatMapSet"; +import type { Collection } from "../struct/Collection"; + +interface DownloadManagerEvents { + downloaded: (beatMapSet: BeatMapSet) => void; + error: (beatMapSet: BeatMapSet, e: unknown) => void; + end: (beatMapSet: BeatMapSet[]) => void; + retrying: (beatMapSet: BeatMapSet) => void; + downloading: (beatMapSet: BeatMapSet) => void; +} + +export declare interface DownloadManager { + on( + event: U, + listener: DownloadManagerEvents[U] + ): this; + + emit( + event: U, + ...args: Parameters + ): boolean; +} export class DownloadManager extends EventEmitter { - monitor: Monitor; path: string; parallel: boolean; - impulseRate: number; + concurrency: number; osuMirrorUrl: string; altOsuMirrorUrl: string; + collection: Collection; + not_downloaded: BeatMapSet[] = []; constructor(monitor: Monitor) { super(); - this.monitor = monitor; this.path = _path.join(config.directory, monitor.collection.name); this.parallel = config.parallel; - this.impulseRate = config.dl_impulse_rate; + this.concurrency = config.concurrency; this.osuMirrorUrl = config.osuMirrorApiUrl; this.altOsuMirrorUrl = config.altOsuMirrorUrl; + this.collection = monitor.collection; } public async bulk_download(): Promise { - const ids = Array.from(this.monitor.collection.beatMapSets.keys()); - if (this.parallel) { - // Impulsive download if url length is more then this.impulseRate - ids.length > this.impulseRate - ? await this.impulse(ids, this.impulseRate) - : await Promise.all(ids.map((id) => this._dl(id))); + await this.impulse(); } else { - // Sequential download - for (let i = 0; i < ids.length; i++) await this._dl(ids[i]); + this.collection.beatMapSets.forEach(async (beatMapSet) => { + await this._downloadFile(beatMapSet); + }); } + + this.emit("end", this.not_downloaded); } - private async _dl(id: number): Promise { - let url = this.osuMirrorUrl + id; + private async _downloadFile( + beatMapSet: BeatMapSet, + options: { retries: number; alt?: boolean } = { retries: 3 } // Whether or not use the alternative mirror url + ): Promise { + const url = + (options.alt ? this.altOsuMirrorUrl : this.osuMirrorUrl) + beatMapSet.id; // Request download - console.log("Requesting: " + url); - let res = await fetch(url, { method: "GET" }).catch(); - if (!this.isValidResponse(res)) { - // Sometimes server failed with 503 status code, retrying is needed - console.error("Requesting failed: " + url); - - // Use alternative mirror url - url = this.altOsuMirrorUrl + id; - console.log("Retrying: " + url); - res = await fetch(url, { method: "GET" }).catch(); - - if (!this.isValidResponse(res)) { - console.error("Requesting failed: " + url); - Logger.generateMissingLog(this.path, id.toString()); - return; - } - } - try { + this.emit("downloading", beatMapSet); + + const res = await fetch(url, { method: "GET" }); + if (!res.ok) throw `Status code: ${res.status}`; // Get file name const fileName = this.getFilename(res); - // Check if directory exists - if (!this.checkIfDirectoryExists()) { - console.error("No directory found: " + this.path); - console.log("Use current working directory instead."); - this.path = process.cwd(); - } + if (!this.checkIfDirectoryExists()) this.path = process.cwd(); // Create write stream - const file = createWriteStream(_path.join(this.path, fileName)); - file.on("error", (err) => { - console.error( - "This file could not be downloaded: " + - fileName + - " Due to error: " + - err - ); - }); - - for await (const chunk of res.body!) { - // Write to file - if (!chunk) continue; - file.write(chunk); - } + await new Promise(async (resolve, reject) => { + const file = createWriteStream(_path.join(this.path, fileName)); + file.on("error", (e) => { + reject(e); + }); + + for await (const chunk of res.body!) { + // Write to file + file.write(chunk); + } - // End the write stream - file.end(); + file.end(); + resolve(); + }); - console.log("Downloaded: " + url); + this.emit("downloaded", beatMapSet); } catch (e) { - throw new OcdlError("DOWNLOAD_FAILED", e); + if (options.retries) { + this.emit("retrying", beatMapSet); + this._downloadFile(beatMapSet, { + alt: options.retries === 1, + retries: options.retries - 1, + }); + } else { + this.emit("error", beatMapSet, e); + this.not_downloaded.push(beatMapSet); + } } } @@ -109,10 +119,10 @@ export class DownloadManager extends EventEmitter { if (result) { try { - const replaced = Util.replaceForbiddenChars(result[1]); - const decoded = decodeURIComponent(replaced); + const decoded = decodeURIComponent(result[1]); + const replaced = Util.replaceForbiddenChars(decoded); - fileName = decoded; + fileName = replaced; } catch (e) { throw new OcdlError("FILE_NAME_EXTRACTION_FAILED", e); } @@ -122,35 +132,30 @@ export class DownloadManager extends EventEmitter { return fileName; } - private async impulse(ids: number[], rate: number): Promise { - const downloaded: any[] = []; + private async impulse(): Promise { + const keys = Array.from(this.collection.beatMapSets.keys()); + const loop_amount = Math.ceil( + this.collection.beatMapSets.size / this.concurrency + ); - const perLen = ids.length / rate; - - for (let i = 0; i < perLen; i++) { + for (let i = 0; i < loop_amount; i++) { const promises: Promise[] = []; - /** - * Bursting Rate - */ - const start = i * rate; - const end = (i + 1) * rate; - const inRange = ids.slice(start, end); - const p = inRange.map((id) => this._dl(id)); - promises.push(...p); - - /** - * Resolve Promises - */ - downloaded.push([...(await Promise.all(promises))]); + + // Burst + const start = i * this.concurrency; + const end = (i + 1) * this.concurrency; + const range = keys.slice(start, end); + + for (const id of range) { + const beatMapSet = this.collection.beatMapSets.get(id)!; // always have a value + promises.push(this._downloadFile(beatMapSet)); + } + + await Promise.all(promises); } - return downloaded; } private checkIfDirectoryExists(): boolean { return existsSync(this.path); } - - private isValidResponse(res: Response): boolean { - return res.status === 200 && !!res.body; - } } From 595cd5390be9356ea04b27796aaefd570050648b Mon Sep 17 00:00:00 2001 From: roogue Date: Sat, 7 Jan 2023 12:56:04 +0800 Subject: [PATCH 12/51] Handle download task --- src/core/Main.ts | 50 +++++++++++++++++++++++++++++++++++++++++-- src/core/Monitor.ts | 21 ++++++++++++++---- src/struct/Config.ts | 10 ++++++--- src/struct/Message.ts | 3 ++- 4 files changed, 74 insertions(+), 10 deletions(-) diff --git a/src/core/Main.ts b/src/core/Main.ts index 9dfe669..66c370d 100644 --- a/src/core/Main.ts +++ b/src/core/Main.ts @@ -8,6 +8,8 @@ import _path from "path"; import Util from "../util"; import type Monitor from "./Monitor"; import { config } from "../config"; +import Logger from "./Logger"; +import chalk from "chalk"; export default class Main { monitor: Monitor; @@ -68,7 +70,7 @@ export default class Main { const fetched_collection = this.monitor.condition.fetched_collection + beatmaps.length; this.monitor.setCondition({ fetched_collection }); - // this.monitor.update(); + this.monitor.update(); } catch (e) { throw new OcdlError("REQUEST_DATA_FAILED", e); } @@ -109,7 +111,51 @@ export default class Main { // Download beatmapSet try { const downloadManager = new DownloadManager(this.monitor); - await downloadManager.bulk_download(); + downloadManager.bulk_download(); + + await new Promise((resolve) => { + downloadManager.on("downloading", (beatMapSet) => { + this.monitor.appendLog( + chalk.gray`Downloading [${beatMapSet.id}] ${beatMapSet.title ?? ""}` + ); + this.monitor.update(); + }); + + downloadManager.on("retrying", (beatMapSet) => { + this.monitor.appendLog( + chalk.yellow`Retrying [${beatMapSet.id}] ${beatMapSet.title ?? ""}` + ); + this.monitor.update(); + }); + + downloadManager.on("downloaded", (beatMapSet) => { + const downloaded = this.monitor.condition.downloaded_beatmapset; + this.monitor.setCondition({ downloaded_beatmapset: downloaded + 1 }); + this.monitor.appendLog( + chalk.green`Downloaded [${beatMapSet.id}] ${beatMapSet.title ?? ""}` + ); + this.monitor.update(); + }); + + downloadManager.on("error", (beatMapSet, e) => { + this.monitor.appendLog( + chalk.red`Failed when downloading [${beatMapSet.id}] ${ + beatMapSet.title ?? "" + }, due to error: ${e}` + ); + this.monitor.update(); + }); + + downloadManager.on("end", (beatMapSet) => { + for (let i = 0; i < beatMapSet.length; i++) { + Logger.generateMissingLog( + this.monitor.collection.name, + beatMapSet[i].id.toString() + ); + } + resolve(); + }); + }); } catch (e) { throw e; } diff --git a/src/core/Monitor.ts b/src/core/Monitor.ts index a06ce9e..fffc241 100644 --- a/src/core/Monitor.ts +++ b/src/core/Monitor.ts @@ -11,6 +11,7 @@ interface Condition { retry_mode: boolean; fetched_collection: number; downloaded_beatmapset: number; + download_log: string[]; } export default class Monitor { @@ -32,6 +33,7 @@ export default class Monitor { retry_mode: false, fetched_collection: 0, downloaded_beatmapset: 0, + download_log: [], }; this.version = (require("../../package.json")?.version ?? @@ -49,7 +51,7 @@ export default class Monitor { } update(): void { - if (1 != 1) clear(); + clear(); // Header log(chalk.yellow(`osu-collector-dl v${this.version}`)); log( @@ -88,6 +90,11 @@ export default class Monitor { Object.assign(this.condition, new_condition); } + appendLog(log: string): void { + this.condition.download_log.splice(0, 0, log); + this.condition.download_log.splice(config.logLength, 1); + } + // Task 1 private p_input_id(): void { if (this.condition.retry_input) { @@ -109,7 +116,7 @@ export default class Monitor { log( new Message(Msg.FETCH_DATA, { amount: this.condition.fetched_collection.toString(), - length: beatmaps_length, + total: beatmaps_length, }).toString() ); } @@ -128,13 +135,19 @@ export default class Monitor { ); } - // Task 5 + // Task 6 private p_download(): void { log( - new Message(Msg.GENERATE_OSDB, { + new Message(Msg.DOWNLOAD_FILE, { amount: this.condition.downloaded_beatmapset.toString(), total: this.collection.beatMapSets.size.toString(), }).toString() ); + + log( + new Message(Msg.DOWNLOAD_LOG, { + log: this.condition.download_log.join("\n"), + }).toString() + ); } } diff --git a/src/struct/Config.ts b/src/struct/Config.ts index 72a1269..aa55731 100644 --- a/src/struct/Config.ts +++ b/src/struct/Config.ts @@ -6,9 +6,10 @@ export default class Config { osuCollectorApiUrl: string; osuMirrorApiUrl: string; altOsuMirrorUrl: string; - dl_impulse_rate: number; + concurrency: number; directory: string; mode: number; + logLength: number; static readonly configFilePath = "./config.json"; constructor(object?: Record) { @@ -21,12 +22,15 @@ export default class Config { // alt Osu mirror url this.altOsuMirrorUrl = "https://kitsu.moe/api/d/"; + // The length of log when downloading beatmapsets + this.logLength = 10; + // Whether download process should be done in parallel this.parallel = Util.isBoolean(object?.parallel) ? object!.parallel : true; // How many urls should be downloaded in parallel at once - this.dl_impulse_rate = !isNaN(Number(object?.dl_impulse_rate)) - ? Number(object!.dl_impulse_rate) + this.concurrency = !isNaN(Number(object?.concurrency)) + ? Number(object!.concurrency) : 10; // Directory to save beatmaps diff --git a/src/struct/Message.ts b/src/struct/Message.ts index 55d4765..a405cfe 100644 --- a/src/struct/Message.ts +++ b/src/struct/Message.ts @@ -32,5 +32,6 @@ export enum Msg { GENERATE_OSDB = "Generating {{name}}.osdb file", - DOWNLOAD_SONG = "Downloading [ {{amount}}/{{total}} ] beatmap set...", + DOWNLOAD_FILE = "Downloading [ {{amount}}/{{total}} ] beatmap set...", + DOWNLOAD_LOG = "Logs: \n{{log}}" } From 7e2442de6150339f3ef1ddd3a351f069603d8124 Mon Sep 17 00:00:00 2001 From: roogue Date: Sat, 7 Jan 2023 17:47:07 +0800 Subject: [PATCH 13/51] Added FAQ section --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 51c44ae..ab33862 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,20 @@ Below is the data stored in config.json, along with explanations of each setting - `mode`: The mode in which the program should operate. Set to `1` to only download the beatmap sets, or `2` to also generate a .osdb file during the download process. You can also specify the mode at the terminal. +## FAQ + +### It says "Retrying" during the download process, am I doing anything wrong? +> No, you are not doing anything wrong. It is normal for API requests to sometimes fail due to factors such as rate limiting and internet connection issues. The script has a built-in retrying process that will handle these issues automatically. It is expected to see the "Retrying" message during the download process. + +### I want the beatmaps to be automatically added to my collections. Is that possible? +> Unfortunately, this feature will not be implemented as directly modifying your personal osu! folder is risky and could potentially result in corrupted files. It is recommended to use Collection Manager (CM) by Piotrekol to modify your collection for more stable functionality. + +### Why won't my program even start? The program shuts off right after I opened it. +> There could be several reasons why your program is not starting. One potential cause is that you have incorrectly edited the config.json file, such as forgetting to include double quotes around the directory path. If you are not sure what the problem is, try reinstalling the program to see if that resolves the issue. + +### I have tried following the FAQ above, but it didn't solve my problem. The problem I am experiencing is not listed in the FAQ. +> If you are experiencing a problem that is not covered in the FAQ and you need assistance, it is welcome to open an issue on the [Issue Page](https://github.com/roogue/osu-collector-dl/issues). After navigating to the issue page, click the green "New issue" button on the page and follow the instructions to describe your problem in as much detail as possible. This will allow the maintainers of the project to better understand and help troubleshoot the issue you are experiencing. + ## License This project is licensed under the MIT License. See the [LICENSE](https://choosealicense.com/licenses/mit/) file for details. \ No newline at end of file From 037f0bc674da33e6e543b1d101864b72960087bf Mon Sep 17 00:00:00 2001 From: roogue Date: Sat, 7 Jan 2023 17:47:54 +0800 Subject: [PATCH 14/51] Fix minor issue --- src/core/Main.ts | 92 +++++++++++++++++++++++++----------------------- 1 file changed, 48 insertions(+), 44 deletions(-) diff --git a/src/core/Main.ts b/src/core/Main.ts index 66c370d..b4e7dae 100644 --- a/src/core/Main.ts +++ b/src/core/Main.ts @@ -1,6 +1,5 @@ import { DownloadManager } from "./DownloadManager"; import { request } from "undici"; -import type Config from "../struct/Config"; import OsdbGenerator from "./OsdbGenerator"; import OcdlError from "../struct/OcdlError"; import { existsSync, mkdirSync } from "fs"; @@ -10,10 +9,10 @@ import type Monitor from "./Monitor"; import { config } from "../config"; import Logger from "./Logger"; import chalk from "chalk"; +import { Msg } from "../struct/Message"; export default class Main { monitor: Monitor; - config: Config = config; collectionApiUrl: string; collectionApiUrlV2: string; @@ -22,11 +21,11 @@ export default class Main { const id = monitor.collection.id; // Quick hand api url for faster fetching - this.collectionApiUrl = this.config.osuCollectorApiUrl + id.toString(); + this.collectionApiUrl = config.osuCollectorApiUrl + id.toString(); // Api url for full information this.collectionApiUrlV2 = - this.config.osuCollectorApiUrl + id.toString() + "/beatmapsV2"; + config.osuCollectorApiUrl + id.toString() + "/beatmapsV2"; } async run(): Promise { @@ -40,7 +39,7 @@ export default class Main { this.monitor.update(); // Fetch full data if user wants generate osdb file - if (this.config.mode === 2) { + if (config.mode === 2) { let hasMorePage: boolean = true; let cursor: number = 0; @@ -50,16 +49,15 @@ export default class Main { if (v2ResponseData instanceof OcdlError) throw v2ResponseData; try { - const { hasMore, nextPageCursor, beatmaps } = v2ResponseData; - - const und = Util.checkUndefined({ - hasMore, - nextPageCursor, - beatmaps, - }); + const und = Util.checkUndefined(v2ResponseData, [ + "hasMore", + "nextPageCursor", + "beatmaps", + ]); if (und) throw new OcdlError("CORRUPTED_RESPONSE", `${und} is required`); + const { hasMore, nextPageCursor, beatmaps } = v2ResponseData; // Resolve all required data this.monitor.collection.resolveFullData(beatmaps); @@ -84,7 +82,7 @@ export default class Main { // Create folder try { responseData.name = Util.replaceForbiddenChars(responseData.name); - const path = _path.join(this.config.directory, responseData.name); + const path = _path.join(config.directory, responseData.name); if (!existsSync(path)) mkdirSync(path); } catch (e) { throw new OcdlError("FOLDER_GENERATION_FAILED", e); @@ -95,7 +93,7 @@ export default class Main { this.monitor.update(); // Generate .osdb file - if (this.config.mode === 2) { + if (config.mode === 2) { try { const generator = new OsdbGenerator(this.monitor); await generator.writeOsdb(); @@ -109,43 +107,49 @@ export default class Main { this.monitor.update(); // Download beatmapSet + // This is added for people who don't want to download beatmaps + console.log(Msg.PRE_DOWNLOAD); + await new Promise((r) => setTimeout(r, 3e3)); + try { const downloadManager = new DownloadManager(this.monitor); - downloadManager.bulk_download(); + downloadManager.on("downloading", (beatMapSet) => { + this.monitor.appendLog( + chalk.gray`Downloading [${beatMapSet.id}] ${beatMapSet.title ?? ""}` + ); + this.monitor.update(); + }); - await new Promise((resolve) => { - downloadManager.on("downloading", (beatMapSet) => { - this.monitor.appendLog( - chalk.gray`Downloading [${beatMapSet.id}] ${beatMapSet.title ?? ""}` - ); - this.monitor.update(); - }); + downloadManager.on("retrying", (beatMapSet) => { + this.monitor.appendLog( + chalk.yellow`Retrying [${beatMapSet.id}] ${beatMapSet.title ?? ""}` + ); + this.monitor.update(); + }); - downloadManager.on("retrying", (beatMapSet) => { - this.monitor.appendLog( - chalk.yellow`Retrying [${beatMapSet.id}] ${beatMapSet.title ?? ""}` - ); - this.monitor.update(); + downloadManager.on("downloaded", (beatMapSet) => { + const downloaded = this.monitor.condition.downloaded_beatmapset; + this.monitor.setCondition({ + downloaded_beatmapset: downloaded + 1, }); + this.monitor.appendLog( + chalk.green`Downloaded [${beatMapSet.id}] ${beatMapSet.title ?? ""}` + ); + this.monitor.update(); + }); - downloadManager.on("downloaded", (beatMapSet) => { - const downloaded = this.monitor.condition.downloaded_beatmapset; - this.monitor.setCondition({ downloaded_beatmapset: downloaded + 1 }); - this.monitor.appendLog( - chalk.green`Downloaded [${beatMapSet.id}] ${beatMapSet.title ?? ""}` - ); - this.monitor.update(); - }); + downloadManager.on("error", (beatMapSet, e) => { + this.monitor.appendLog( + chalk.red`Failed when downloading [${beatMapSet.id}] ${ + beatMapSet.title ?? "" + }, due to error: ${e}` + ); + this.monitor.update(); + }); - downloadManager.on("error", (beatMapSet, e) => { - this.monitor.appendLog( - chalk.red`Failed when downloading [${beatMapSet.id}] ${ - beatMapSet.title ?? "" - }, due to error: ${e}` - ); - this.monitor.update(); - }); + downloadManager.bulk_download(); + await new Promise((resolve) => { downloadManager.on("end", (beatMapSet) => { for (let i = 0; i < beatMapSet.length; i++) { Logger.generateMissingLog( @@ -160,7 +164,7 @@ export default class Main { throw e; } - this.monitor.freeze("Download finished"); + this.monitor.freeze("\nDownload finished"); return; } From 37bb2ee3a83c01af271debadc49f38424bf1807e Mon Sep 17 00:00:00 2001 From: roogue Date: Sat, 7 Jan 2023 17:48:27 +0800 Subject: [PATCH 15/51] Enhance typings --- src/struct/BeatMap.ts | 6 +++-- src/struct/BeatMapSet.ts | 17 +++++++------ src/struct/Collection.ts | 53 ++++++++++++++++++++-------------------- src/types.ts | 32 +++++++++--------------- 4 files changed, 52 insertions(+), 56 deletions(-) diff --git a/src/struct/BeatMap.ts b/src/struct/BeatMap.ts index 13b5f84..129cdd4 100644 --- a/src/struct/BeatMap.ts +++ b/src/struct/BeatMap.ts @@ -1,3 +1,4 @@ +import type { BeatMapType } from "../types"; import Util from "../util"; import OcdlError from "./OcdlError"; @@ -12,10 +13,11 @@ export class BeatMap { difficulty_rating?: number; constructor(jsonData: Record) { - const { id, checksum } = jsonData; - const und = Util.checkUndefined({ id, checksum }); + const und = Util.checkUndefined(jsonData, ["id", "checksum"]); if (und) throw new OcdlError("CORRUPTED_RESPONSE", `${und} is required`); + const { id, checksum } = jsonData as BeatMapType; + this.id = id; this.checksum = checksum; } diff --git a/src/struct/BeatMapSet.ts b/src/struct/BeatMapSet.ts index b60a72f..3344eda 100644 --- a/src/struct/BeatMapSet.ts +++ b/src/struct/BeatMapSet.ts @@ -1,3 +1,4 @@ +import type { BeatMapSetType, BeatMapType } from "../types"; import Util from "../util"; import { BeatMap } from "./BeatMap"; import OcdlError from "./OcdlError"; @@ -8,24 +9,24 @@ export class BeatMapSet { beatMaps: Map; // nullable - title?: string - artist?: string + title?: string; + artist?: string; constructor(jsonData: Record) { - const { id, beatmaps } = jsonData; - const und = Util.checkUndefined({ id, beatmaps }); + const und = Util.checkUndefined(jsonData, ["id", "beatmaps"]); if (und) throw new OcdlError("CORRUPTED_RESPONSE", `${und} is required`); + const { id, beatmaps } = jsonData as BeatMapSetType; + this.id = id; this.beatMaps = this._resolveBeatMaps(beatmaps); } - private _resolveBeatMaps(array: Record[]) { + private _resolveBeatMaps(beatMapJson: BeatMapType[]) { const resolvedData = new Map(); - const unresolvedData = array; - for (let i = 0; i < unresolvedData.length; i++) { + for (let i = 0; i < beatMapJson.length; i++) { try { - const map = new BeatMap(unresolvedData[i]); + const map = new BeatMap(beatMapJson[i]); resolvedData.set(map.id, map); } catch (e) { throw new OcdlError("CORRUPTED_RESPONSE", e); diff --git a/src/struct/Collection.ts b/src/struct/Collection.ts index 08003b5..05bbfc2 100644 --- a/src/struct/Collection.ts +++ b/src/struct/Collection.ts @@ -1,7 +1,7 @@ import OcdlError from "./OcdlError"; import { BeatMapSet } from "./BeatMapSet"; import Util from "../util"; -import { ModeByte } from "../types"; +import { BeatMapSetType, CollectionType, FullBeatMapType, ModeByte } from "../types"; export class Collection { beatMapSets: Map = new Map(); @@ -17,16 +17,18 @@ export class Collection { constructor() {} resolveData(jsonData: Record = {}) { - const { id, name, uploader, beatmapsets, beatmapCount } = jsonData; - const und = Util.checkUndefined({ - id, - name, - uploader, - beatmapsets, - beatmapCount, - }); + const und = Util.checkUndefined(jsonData, [ + "id", + "name", + "uploader", + "beatmapsets", + "beatmapCount", + ]); if (und) throw new OcdlError("CORRUPTED_RESPONSE", `${und} is required`); + const { id, name, uploader, beatmapsets, beatmapCount } = + jsonData as CollectionType; + this.id = id; this.name = name; this.uploader = uploader; @@ -34,22 +36,22 @@ export class Collection { this.beatMapCount = beatmapCount; } - resolveFullData(array: Record[]): void { - const unresolvedData = array; - if (!unresolvedData.length) + resolveFullData(jsonData: Record[]): void { + if (!jsonData.length) throw new OcdlError("CORRUPTED_RESPONSE", "No beatmap found"); - for (let i = 0; i < unresolvedData.length; i++) { - const { id, mode, difficulty_rating, version, beatmapset } = array[i]; - const und = Util.checkUndefined({ - id, - mode, - difficulty_rating, - version, - beatmapset, - }); + for (let i = 0; i < jsonData.length; i++) { + const und = Util.checkUndefined(jsonData[i], [ + "id", + "mode", + "difficulty_rating", + "version", + "beatmapset", + ]); if (und) throw new OcdlError("CORRUPTED_RESPONSE", `${und} is required`); + const { id, mode, difficulty_rating, version, beatmapset } = jsonData[i] as FullBeatMapType; + const beatMapSet = this.beatMapSets.get(beatmapset.id); if (!beatMapSet) continue; @@ -68,16 +70,15 @@ export class Collection { } private _resolveBeatMapSets( - array: Record[] + beatMapSetJson: BeatMapSetType[] ): Map { const resolvedData = new Map(); - const unresolvedData = array; - if (!unresolvedData.length) + if (!beatMapSetJson.length) throw new OcdlError("CORRUPTED_RESPONSE", "No beatmapset found"); - for (let i = 0; i < unresolvedData.length; i++) { + for (let i = 0; i < beatMapSetJson.length; i++) { try { - const set = new BeatMapSet(unresolvedData[i]); + const set = new BeatMapSet(beatMapSetJson[i]); resolvedData.set(set.id, set); } catch (e) { throw new OcdlError("CORRUPTED_RESPONSE", e); diff --git a/src/types.ts b/src/types.ts index cec6837..11a060a 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,17 +1,17 @@ // * These typings are incomplete, but they are enough to get the app working. // * Reference: https://github.com/roogue/osu-collector-node/blob/main/src/typings/Collection.ts -export interface BeatMap { +export interface BeatMapType { checksum: string; id: number; } -export interface BeatMapSet { - beatmaps: BeatMap[]; +export interface BeatMapSetType { + beatmaps: BeatMapType[]; id: number; } -export interface Collection { - beatmapIds: BeatMap[]; - beatmapsets: BeatMapSet[]; +export interface CollectionType { + beatmapIds: BeatMapType[]; + beatmapsets: BeatMapSetType[]; beatmapCount: number; id: number; name: string; @@ -20,26 +20,18 @@ export interface Collection { }; } -export interface BeatMapV2 { - checksum: string; +export interface FullBeatMapType { id: number; - beatmapset_id: number; - beatmapset: BeatMapSetV2; - version: string; mode: Mode; difficulty_rating: number; + version: string; + beatmapset: FullBeatMapSetType; } -export interface BeatMapSetV2 { - artist: string; +export interface FullBeatMapSetType { + id: number; title: string; - creator: string; -} - -export interface BeatMapV2ResData { - nextPageCursor: number; - hasMore: boolean; - beatmaps: BeatMapV2[]; + artist: string; } export type Mode = "taiko" | "osu" | "fruits" | "mania"; From e2ff0c1593e0e7362139f7ea0ab32bd569ef589c Mon Sep 17 00:00:00 2001 From: roogue Date: Sat, 7 Jan 2023 17:48:51 +0800 Subject: [PATCH 16/51] Update Message.ts --- src/struct/Message.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/struct/Message.ts b/src/struct/Message.ts index a405cfe..cc4c7f0 100644 --- a/src/struct/Message.ts +++ b/src/struct/Message.ts @@ -32,6 +32,7 @@ export enum Msg { GENERATE_OSDB = "Generating {{name}}.osdb file", + PRE_DOWNLOAD = "Download will be start in 3 seconds...", DOWNLOAD_FILE = "Downloading [ {{amount}}/{{total}} ] beatmap set...", - DOWNLOAD_LOG = "Logs: \n{{log}}" + DOWNLOAD_LOG = "{{log}}", } From 7abd82be8077c1a45e1bcbda6e6af2e7b1c964c7 Mon Sep 17 00:00:00 2001 From: roogue Date: Sat, 7 Jan 2023 17:49:07 +0800 Subject: [PATCH 17/51] Optimize util function --- src/util.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/util.ts b/src/util.ts index d6f6526..6f7a244 100644 --- a/src/util.ts +++ b/src/util.ts @@ -8,9 +8,14 @@ export default class Util { return str.replace(regex, ""); } - static checkUndefined(obj: Record): string | null { - for (const [key, value] of Object.entries(obj)) { - if (value === undefined) return key; + static checkUndefined( + obj: Record, + fields: string[] + ): string | null { + for (const field of fields) { + if (!obj.hasOwnProperty(field)) { + return field; + } } return null; } From d0f664816d97af2211b98dad5f67475c73412e41 Mon Sep 17 00:00:00 2001 From: roogue Date: Sat, 7 Jan 2023 18:03:10 +0800 Subject: [PATCH 18/51] Handling error from config --- src/config.ts | 2 +- src/struct/Config.ts | 29 ++++++++++++++++++++--------- src/struct/OcdlError.ts | 1 + 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/src/config.ts b/src/config.ts index 11f1824..7c8576c 100644 --- a/src/config.ts +++ b/src/config.ts @@ -5,5 +5,5 @@ const filePath = Config.configFilePath; // If config file doesn't exist, create one export const config: Config = existsSync(filePath) - ? new Config(JSON.parse(readFileSync(filePath, "utf8"))) + ? new Config(readFileSync(filePath, "utf8")) : Config.generateConfig(); \ No newline at end of file diff --git a/src/struct/Config.ts b/src/struct/Config.ts index aa55731..c423785 100644 --- a/src/struct/Config.ts +++ b/src/struct/Config.ts @@ -1,5 +1,7 @@ import { existsSync, writeFileSync } from "fs"; +import Logger from "../core/Logger"; import Util from "../util"; +import OcdlError from "./OcdlError"; export default class Config { parallel: boolean; @@ -12,7 +14,16 @@ export default class Config { logLength: number; static readonly configFilePath = "./config.json"; - constructor(object?: Record) { + constructor(contents?: string) { + let config: Record = {}; + if (contents) { + try { + config = JSON.parse(contents); + } catch (e) { + throw Logger.generateErrorLog(new OcdlError("INVALID_CONFIG", e)); + } + } + // Osucollector's base url this.osuCollectorApiUrl = "https://osucollector.com/api/collections/"; @@ -26,23 +37,23 @@ export default class Config { this.logLength = 10; // Whether download process should be done in parallel - this.parallel = Util.isBoolean(object?.parallel) ? object!.parallel : true; + this.parallel = Util.isBoolean(config.parallel) ? config.parallel : true; // How many urls should be downloaded in parallel at once - this.concurrency = !isNaN(Number(object?.concurrency)) - ? Number(object!.concurrency) + this.concurrency = !isNaN(Number(config.concurrency)) + ? Number(config.concurrency) : 10; // Directory to save beatmaps - this.directory = object?.directory - ? String(object?.directory) + this.directory = config.directory + ? String(config.directory) : process.cwd(); // Mode // 1: Download BeatmapSet // 2: Download BeatmapSet + Generate .osdb - if (object?.mode) { - const mode = Number(object.mode); + if (config.mode) { + const mode = Number(config.mode); // Mode should be 1 or 2 ![1, 2].includes(mode) ? (this.mode = 1) : (this.mode = mode); } else { @@ -56,7 +67,7 @@ export default class Config { Config.configFilePath, JSON.stringify({ parallel: true, - dl_impulse_rate: 5, + concurrency: 5, directory: process.cwd(), mode: 1, }) diff --git a/src/struct/OcdlError.ts b/src/struct/OcdlError.ts index fbbdbdf..065038d 100644 --- a/src/struct/OcdlError.ts +++ b/src/struct/OcdlError.ts @@ -1,4 +1,5 @@ export enum ErrorType { + "INVALID_CONFIG" = "The config is invalid json type", "GET_USER_INPUT_FAILED" = "Error occurred while getting user input", "RESOLVE_JSON_FAILED" = "Error occurred while resolving res body to json", "DOWNLOAD_FAILED" = "Error occurred while downloading beatmapset", From a02c893ee73e4f83574414e2a9a9eefe6d6fdd62 Mon Sep 17 00:00:00 2001 From: roogue Date: Sat, 7 Jan 2023 18:03:13 +0800 Subject: [PATCH 19/51] Update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4066a31..f73fb30 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "osu-collector-dl", - "version": "2.6.4", + "version": "3.0.0", "main": "./dist/index.js", "scripts": { "build": "yarn run build-app && yarn run cp-build", From cd6d67d3f6f4c4919c361bc2da69003135e356de Mon Sep 17 00:00:00 2001 From: roogue Date: Sat, 7 Jan 2023 19:19:27 +0800 Subject: [PATCH 20/51] Use collection instance instead --- src/core/DownloadManager.ts | 7 ++-- src/core/Main.ts | 4 +- src/core/OsdbGenerator.ts | 81 +++++++++++++++++++------------------ 3 files changed, 46 insertions(+), 46 deletions(-) diff --git a/src/core/DownloadManager.ts b/src/core/DownloadManager.ts index ec5eeda..a44864b 100644 --- a/src/core/DownloadManager.ts +++ b/src/core/DownloadManager.ts @@ -4,7 +4,6 @@ import _path from "path"; import OcdlError from "../struct/OcdlError"; import Util from "../util"; import EventEmitter from "events"; -import type Monitor from "./Monitor"; import { config } from "../config"; import type { BeatMapSet } from "../struct/BeatMapSet"; import type { Collection } from "../struct/Collection"; @@ -38,14 +37,14 @@ export class DownloadManager extends EventEmitter { collection: Collection; not_downloaded: BeatMapSet[] = []; - constructor(monitor: Monitor) { + constructor(collection: Collection) { super(); - this.path = _path.join(config.directory, monitor.collection.name); + this.path = _path.join(config.directory, collection.name); this.parallel = config.parallel; this.concurrency = config.concurrency; this.osuMirrorUrl = config.osuMirrorApiUrl; this.altOsuMirrorUrl = config.altOsuMirrorUrl; - this.collection = monitor.collection; + this.collection = collection; } public async bulk_download(): Promise { diff --git a/src/core/Main.ts b/src/core/Main.ts index b4e7dae..5897ff2 100644 --- a/src/core/Main.ts +++ b/src/core/Main.ts @@ -95,7 +95,7 @@ export default class Main { // Generate .osdb file if (config.mode === 2) { try { - const generator = new OsdbGenerator(this.monitor); + const generator = new OsdbGenerator(this.monitor.collection); await generator.writeOsdb(); } catch (e) { throw new OcdlError("GENERATE_OSDB_FAILED", e); @@ -112,7 +112,7 @@ export default class Main { await new Promise((r) => setTimeout(r, 3e3)); try { - const downloadManager = new DownloadManager(this.monitor); + const downloadManager = new DownloadManager(this.monitor.collection); downloadManager.on("downloading", (beatMapSet) => { this.monitor.appendLog( chalk.gray`Downloading [${beatMapSet.id}] ${beatMapSet.title ?? ""}` diff --git a/src/core/OsdbGenerator.ts b/src/core/OsdbGenerator.ts index b3ff7d5..9fe8f96 100644 --- a/src/core/OsdbGenerator.ts +++ b/src/core/OsdbGenerator.ts @@ -2,20 +2,23 @@ import { BinaryWriter, File, IFile } from "csbinary"; import { openSync, writeFileSync } from "fs"; import { config } from "../config"; import _path from "path"; -import type Monitor from "./Monitor"; +import type { Collection } from "../struct/Collection"; +import Util from "../util"; export default class OsdbGenerator { filePath: string; fileName: string; file: IFile; writer: BinaryWriter; - monitor: Monitor; + collection: Collection; - constructor(monitor: Monitor) { - this.fileName = monitor.collection.name + ".osdb"; + constructor(collection: Collection) { + const collectionName = Util.replaceForbiddenChars(collection.name); + + this.fileName = collectionName + ".osdb"; this.filePath = _path.join( config.directory, - monitor.collection.name, // Folder name + collectionName, // Folder name this.fileName ); // Create file @@ -25,7 +28,7 @@ export default class OsdbGenerator { this.writer = new BinaryWriter(this.file); - this.monitor = monitor; + this.collection = collection; } // * Refer https://github.com/Piotrekol/CollectionManager/blob/master/CollectionManagerDll/Modules/FileIO/FileCollections/OsdbCollectionHandler.cs#L89 @@ -38,47 +41,45 @@ export default class OsdbGenerator { this.writer.writeDouble(this.toOADate(new Date())); // Editor - this.writer.writeString(this.monitor.collection.uploader.username); + this.writer.writeString(this.collection.uploader.username); // Num of collections this.writer.writeInt32(1); // Always 1 // Name - this.writer.writeString(this.monitor.collection.name); + this.writer.writeString(this.collection.name); // Beatmap count - this.writer.writeInt32(this.monitor.collection.beatMapCount); - - this.monitor.collection.beatMapSets.forEach( - (beatMapSet, beatMapSetId) => { - beatMapSet.beatMaps.forEach((beatmap, beatMapId) => { - // BeatmapId - this.writer.writeInt32(beatMapId); - - // BeatmapSetId - this.writer.writeInt32(beatMapSetId); - - // Artist - this.writer.writeString(beatMapSet.artist ?? "Unknown"); - // Title - this.writer.writeString(beatMapSet.title ?? "Unknown"); - // Version - this.writer.writeString(beatmap.version ?? "Unknown"); - - // Md5 - this.writer.writeString(beatmap.checksum); - - // User comment - this.writer.writeString(""); - - // Play mode - this.writer.writeByte(beatmap.mode ?? 0); - - // Mod PP Star - this.writer.writeDouble(beatmap.difficulty_rating ?? 0); - }); - } - ); + this.writer.writeInt32(this.collection.beatMapCount); + + this.collection.beatMapSets.forEach((beatMapSet, beatMapSetId) => { + beatMapSet.beatMaps.forEach((beatmap, beatMapId) => { + // BeatmapId + this.writer.writeInt32(beatMapId); + + // BeatmapSetId + this.writer.writeInt32(beatMapSetId); + + // Artist + this.writer.writeString(beatMapSet.artist ?? "Unknown"); + // Title + this.writer.writeString(beatMapSet.title ?? "Unknown"); + // Version + this.writer.writeString(beatmap.version ?? "Unknown"); + + // Md5 + this.writer.writeString(beatmap.checksum); + + // User comment + this.writer.writeString(""); + + // Play mode + this.writer.writeByte(beatmap.mode ?? 0); + + // Mod PP Star + this.writer.writeDouble(beatmap.difficulty_rating ?? 0); + }); + }); // Map with hash this.writer.writeInt32(0); // Always 0 From 134f9dc3022d78ea34b5f000c59a741698caad40 Mon Sep 17 00:00:00 2001 From: roogue Date: Sat, 7 Jan 2023 19:20:15 +0800 Subject: [PATCH 21/51] Use array reduce instead of loop --- src/struct/BeatMapSet.ts | 13 ++++++------- src/struct/Collection.ts | 25 ++++++++++++++----------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/struct/BeatMapSet.ts b/src/struct/BeatMapSet.ts index 3344eda..5532358 100644 --- a/src/struct/BeatMapSet.ts +++ b/src/struct/BeatMapSet.ts @@ -22,16 +22,15 @@ export class BeatMapSet { this.beatMaps = this._resolveBeatMaps(beatmaps); } - private _resolveBeatMaps(beatMapJson: BeatMapType[]) { - const resolvedData = new Map(); - for (let i = 0; i < beatMapJson.length; i++) { + private _resolveBeatMaps(beatMapJson: BeatMapType[]): Map { + return beatMapJson.reduce((acc, current) => { try { - const map = new BeatMap(beatMapJson[i]); - resolvedData.set(map.id, map); + const map = new BeatMap(current); + acc.set(map.id, map); + return acc; } catch (e) { throw new OcdlError("CORRUPTED_RESPONSE", e); } - } - return resolvedData; + }, new Map()); } } diff --git a/src/struct/Collection.ts b/src/struct/Collection.ts index 05bbfc2..8cee7b9 100644 --- a/src/struct/Collection.ts +++ b/src/struct/Collection.ts @@ -1,7 +1,12 @@ import OcdlError from "./OcdlError"; import { BeatMapSet } from "./BeatMapSet"; import Util from "../util"; -import { BeatMapSetType, CollectionType, FullBeatMapType, ModeByte } from "../types"; +import { + BeatMapSetType, + CollectionType, + FullBeatMapType, + ModeByte, +} from "../types"; export class Collection { beatMapSets: Map = new Map(); @@ -50,7 +55,9 @@ export class Collection { ]); if (und) throw new OcdlError("CORRUPTED_RESPONSE", `${und} is required`); - const { id, mode, difficulty_rating, version, beatmapset } = jsonData[i] as FullBeatMapType; + const { id, mode, difficulty_rating, version, beatmapset } = jsonData[ + i + ] as FullBeatMapType; const beatMapSet = this.beatMapSets.get(beatmapset.id); if (!beatMapSet) continue; @@ -72,18 +79,14 @@ export class Collection { private _resolveBeatMapSets( beatMapSetJson: BeatMapSetType[] ): Map { - const resolvedData = new Map(); - if (!beatMapSetJson.length) - throw new OcdlError("CORRUPTED_RESPONSE", "No beatmapset found"); - - for (let i = 0; i < beatMapSetJson.length; i++) { + return beatMapSetJson.reduce((acc, current) => { try { - const set = new BeatMapSet(beatMapSetJson[i]); - resolvedData.set(set.id, set); + const map = new BeatMapSet(current); + acc.set(map.id, map); + return acc; } catch (e) { throw new OcdlError("CORRUPTED_RESPONSE", e); } - } - return resolvedData; + }, new Map()); } } From a036ad5d054436f83e212372c2a446223ba86b92 Mon Sep 17 00:00:00 2001 From: roogue Date: Sat, 7 Jan 2023 19:20:17 +0800 Subject: [PATCH 22/51] Update Message.ts --- src/struct/Message.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/struct/Message.ts b/src/struct/Message.ts index cc4c7f0..452d7f1 100644 --- a/src/struct/Message.ts +++ b/src/struct/Message.ts @@ -32,7 +32,7 @@ export enum Msg { GENERATE_OSDB = "Generating {{name}}.osdb file", - PRE_DOWNLOAD = "Download will be start in 3 seconds...", + PRE_DOWNLOAD = "Download will starts automatically...", DOWNLOAD_FILE = "Downloading [ {{amount}}/{{total}} ] beatmap set...", DOWNLOAD_LOG = "{{log}}", } From 2621578307e696f03ef87b391c1f26c0bbb662e1 Mon Sep 17 00:00:00 2001 From: roogue Date: Sat, 7 Jan 2023 19:20:31 +0800 Subject: [PATCH 23/51] More readable code --- src/struct/Config.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/struct/Config.ts b/src/struct/Config.ts index c423785..10e13ff 100644 --- a/src/struct/Config.ts +++ b/src/struct/Config.ts @@ -52,13 +52,9 @@ export default class Config { // Mode // 1: Download BeatmapSet // 2: Download BeatmapSet + Generate .osdb - if (config.mode) { - const mode = Number(config.mode); - // Mode should be 1 or 2 - ![1, 2].includes(mode) ? (this.mode = 1) : (this.mode = mode); - } else { - this.mode = 1; - } + config.mode && this.isValidMode(config.mode) + ? (this.mode = config.mode) + : (this.mode = 1); } static generateConfig(): Config { @@ -79,4 +75,8 @@ export default class Config { private static checkIfConfigFileExist(): boolean { return existsSync(Config.configFilePath); } + + private isValidMode(mode: any): mode is 1 | 2 { + return typeof mode === "number" && [1, 2].includes(mode); + } } From ced04c09aeae751cbda1ac265bc92097c325cf63 Mon Sep 17 00:00:00 2001 From: roogue Date: Sat, 7 Jan 2023 19:20:44 +0800 Subject: [PATCH 24/51] Log the error first before closing --- src/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/index.ts b/src/index.ts index 9ea3cff..f4f2d03 100644 --- a/src/index.ts +++ b/src/index.ts @@ -74,7 +74,7 @@ const isOnline = async (): Promise => { try { await main.run(); } catch (e) { - console.error(e); - if (e instanceof OcdlError) return Logger.generateErrorLog(e); + if (e instanceof OcdlError) Logger.generateErrorLog(e); + monitor.freeze("An error occurred: " + e, true); } })(); From dcdc87be382e8e0fe4f4cc087430f5e1e3a4e251 Mon Sep 17 00:00:00 2001 From: roogue Date: Sat, 7 Jan 2023 19:22:46 +0800 Subject: [PATCH 25/51] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ab33862..8fa473d 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Below is the data stored in config.json, along with explanations of each setting > No, you are not doing anything wrong. It is normal for API requests to sometimes fail due to factors such as rate limiting and internet connection issues. The script has a built-in retrying process that will handle these issues automatically. It is expected to see the "Retrying" message during the download process. ### I want the beatmaps to be automatically added to my collections. Is that possible? -> Unfortunately, this feature will not be implemented as directly modifying your personal osu! folder is risky and could potentially result in corrupted files. It is recommended to use Collection Manager (CM) by Piotrekol to modify your collection for more stable functionality. +> Unfortunately, this feature will not be implemented as directly modifying your personal osu! folder is risky and could potentially result in corrupted files. It is recommended to use [Collection Manager](https://github.com/Piotrekol/CollectionManager) (CM) by Piotrekol to modify your collection for more stable functionality. ### Why won't my program even start? The program shuts off right after I opened it. > There could be several reasons why your program is not starting. One potential cause is that you have incorrectly edited the config.json file, such as forgetting to include double quotes around the directory path. If you are not sure what the problem is, try reinstalling the program to see if that resolves the issue. From e78114f91c1d28ebbb6aba700ee12643d5e5b430 Mon Sep 17 00:00:00 2001 From: roogue Date: Sat, 7 Jan 2023 19:36:35 +0800 Subject: [PATCH 26/51] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8fa473d..2d76843 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Below is the data stored in config.json, along with explanations of each setting ```json { "parallel": true, - "concurrency": 10, + "concurrency": 5, "directory": "", "mode": 1 } From 76ae78ec525fc081191287e7100487b42bbe13fb Mon Sep 17 00:00:00 2001 From: roogue Date: Sun, 8 Jan 2023 18:17:01 +0800 Subject: [PATCH 27/51] Rename Main to Worker --- src/config.ts | 9 --- src/core/{Main.ts => Worker.ts} | 100 ++++++++++++++++++++++++-------- 2 files changed, 75 insertions(+), 34 deletions(-) delete mode 100644 src/config.ts rename src/core/{Main.ts => Worker.ts} (68%) diff --git a/src/config.ts b/src/config.ts deleted file mode 100644 index 7c8576c..0000000 --- a/src/config.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { existsSync, readFileSync } from "fs"; -import Config from "./struct/Config"; - -const filePath = Config.configFilePath; - -// If config file doesn't exist, create one -export const config: Config = existsSync(filePath) - ? new Config(readFileSync(filePath, "utf8")) - : Config.generateConfig(); \ No newline at end of file diff --git a/src/core/Main.ts b/src/core/Worker.ts similarity index 68% rename from src/core/Main.ts rename to src/core/Worker.ts index 5897ff2..5291d31 100644 --- a/src/core/Main.ts +++ b/src/core/Worker.ts @@ -5,41 +5,86 @@ import OcdlError from "../struct/OcdlError"; import { existsSync, mkdirSync } from "fs"; import _path from "path"; import Util from "../util"; -import type Monitor from "./Monitor"; -import { config } from "../config"; +import Monitor from "./Monitor"; import Logger from "./Logger"; import chalk from "chalk"; -import { Msg } from "../struct/Message"; +import { Message, Msg } from "../struct/Message"; +import Manager from "./Manager"; -export default class Main { +export default class Worker extends Manager { monitor: Monitor; - collectionApiUrl: string; - collectionApiUrlV2: string; - constructor(monitor: Monitor) { - this.monitor = monitor; - - const id = monitor.collection.id; - // Quick hand api url for faster fetching - this.collectionApiUrl = config.osuCollectorApiUrl + id.toString(); - - // Api url for full information - this.collectionApiUrlV2 = - config.osuCollectorApiUrl + id.toString() + "/beatmapsV2"; + constructor() { + super(); + this.monitor = new Monitor(); } async run(): Promise { + // Check if internet connection is presence + const onlineStatus = await Util.isOnline(); + if (!onlineStatus) + return this.monitor.freeze( + new Message(Msg.NO_CONNECTION).toString(), + true + ); + + await this.monitor.checkNewVersion(); + + let id: number | null = null; + let mode: number | null = null; + + try { + // task 1 + this.monitor.next(); + + // Get id + while (id === null) { + this.monitor.update(); + + const result = Number( + this.monitor.awaitInput(new Message(Msg.INPUT_ID).toString(), "none") + ); + if (!isNaN(result)) id = result; // check if result is valid + this.monitor.condition.retry_input = true; + } + + Manager.collection.id = id; + + // task 2 + this.monitor.next(); + + // Get mode + while (mode === null) { + this.monitor.update(); + const result = String( + this.monitor.awaitInput( + new Message(Msg.INPUT_MODE, { + mode: Manager.config.mode === 2 ? "Yes" : "No", + }).toString(), + Manager.config.mode.toString() + ) + ); + if (["n", "no", "1"].includes(result)) mode = 1; + if (["y", "yes", "ass", "2"].includes(result)) mode = 2; + this.monitor.condition.retry_mode = true; + } + + Manager.config.mode = mode; + } catch (e) { + throw new OcdlError("GET_USER_INPUT_FAILED", e); + } + // Fetch brief collection info const responseData = await this.fetchCollection(); if (responseData instanceof OcdlError) throw responseData; - this.monitor.collection.resolveData(responseData); + Manager.collection.resolveData(responseData); // Task 3 this.monitor.next(); this.monitor.update(); // Fetch full data if user wants generate osdb file - if (config.mode === 2) { + if (Manager.config.mode === 2) { let hasMorePage: boolean = true; let cursor: number = 0; @@ -59,7 +104,7 @@ export default class Main { const { hasMore, nextPageCursor, beatmaps } = v2ResponseData; // Resolve all required data - this.monitor.collection.resolveFullData(beatmaps); + Manager.collection.resolveFullData(beatmaps); // Set property hasMorePage = hasMore; @@ -67,6 +112,7 @@ export default class Main { const fetched_collection = this.monitor.condition.fetched_collection + beatmaps.length; + this.monitor.setCondition({ fetched_collection }); this.monitor.update(); } catch (e) { @@ -82,7 +128,7 @@ export default class Main { // Create folder try { responseData.name = Util.replaceForbiddenChars(responseData.name); - const path = _path.join(config.directory, responseData.name); + const path = _path.join(Manager.config.directory, responseData.name); if (!existsSync(path)) mkdirSync(path); } catch (e) { throw new OcdlError("FOLDER_GENERATION_FAILED", e); @@ -93,9 +139,9 @@ export default class Main { this.monitor.update(); // Generate .osdb file - if (config.mode === 2) { + if (Manager.config.mode === 2) { try { - const generator = new OsdbGenerator(this.monitor.collection); + const generator = new OsdbGenerator(); await generator.writeOsdb(); } catch (e) { throw new OcdlError("GENERATE_OSDB_FAILED", e); @@ -112,7 +158,7 @@ export default class Main { await new Promise((r) => setTimeout(r, 3e3)); try { - const downloadManager = new DownloadManager(this.monitor.collection); + const downloadManager = new DownloadManager(); downloadManager.on("downloading", (beatMapSet) => { this.monitor.appendLog( chalk.gray`Downloading [${beatMapSet.id}] ${beatMapSet.title ?? ""}` @@ -153,7 +199,7 @@ export default class Main { downloadManager.on("end", (beatMapSet) => { for (let i = 0; i < beatMapSet.length; i++) { Logger.generateMissingLog( - this.monitor.collection.name, + Manager.collection.name, beatMapSet[i].id.toString() ); } @@ -174,7 +220,11 @@ export default class Main { cursor: number = 0 ): Promise | OcdlError> { // Check version of collection - const url = v2 ? this.collectionApiUrlV2 : this.collectionApiUrl; + const url = + Manager.config.osuCollectorApiUrl + + Manager.collection.id.toString() + + (v2 ? "/beatmapsV2" : ""); + const query: Record = // Query is needed for V2 collection v2 ? { From d9de4ab3c32d2cb8b572a989877309a2d73d3243 Mon Sep 17 00:00:00 2001 From: roogue Date: Sun, 8 Jan 2023 18:17:26 +0800 Subject: [PATCH 28/51] Restructure Config.ts --- src/struct/Config.ts | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/struct/Config.ts b/src/struct/Config.ts index 10e13ff..9d1ac37 100644 --- a/src/struct/Config.ts +++ b/src/struct/Config.ts @@ -1,4 +1,5 @@ import { existsSync, writeFileSync } from "fs"; +import path from "path"; import Logger from "../core/Logger"; import Util from "../util"; import OcdlError from "./OcdlError"; @@ -34,7 +35,9 @@ export default class Config { this.altOsuMirrorUrl = "https://kitsu.moe/api/d/"; // The length of log when downloading beatmapsets - this.logLength = 10; + this.logLength = !isNaN(Number(config.logSize)) + ? Number(config.logSize) + : 15; // Whether download process should be done in parallel this.parallel = Util.isBoolean(config.parallel) ? config.parallel : true; @@ -45,16 +48,12 @@ export default class Config { : 10; // Directory to save beatmaps - this.directory = config.directory - ? String(config.directory) - : process.cwd(); + this.directory = this.getPath(config.directory); // Mode // 1: Download BeatmapSet // 2: Download BeatmapSet + Generate .osdb - config.mode && this.isValidMode(config.mode) - ? (this.mode = config.mode) - : (this.mode = 1); + this.mode = this.getMode(config.mode); } static generateConfig(): Config { @@ -76,7 +75,12 @@ export default class Config { return existsSync(Config.configFilePath); } - private isValidMode(mode: any): mode is 1 | 2 { - return typeof mode === "number" && [1, 2].includes(mode); + private getMode(data: any): number { + return data == 1 ? 1 : data == 2 ? 2 : 1; + } + + private getPath(data: any): string { + if (typeof data !== "string") return process.cwd(); + return path.isAbsolute(data) ? data : process.cwd(); } } From 118cbf39be61c2da95ce16dde4e139088d0e28c0 Mon Sep 17 00:00:00 2001 From: roogue Date: Sun, 8 Jan 2023 18:18:17 +0800 Subject: [PATCH 29/51] Added logSize to config --- .gitignore | 1 + README.md | 1 + config.json.example | 1 + 3 files changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index b0afe1f..399bbed 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ #beatmap *.osz +*.osdb # Logs logs diff --git a/README.md b/README.md index 2d76843..5197677 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Below is the data stored in config.json, along with explanations of each setting { "parallel": true, "concurrency": 5, + "logSize": 15, "directory": "", "mode": 1 } diff --git a/config.json.example b/config.json.example index bf43117..1f311ef 100644 --- a/config.json.example +++ b/config.json.example @@ -1,6 +1,7 @@ { "parallel": true, "concurrency": 5, + "logSize": 15, "directory": "", "mode": 1 } From b0618c5d4b70b31f543efd7193b2df8841b92b49 Mon Sep 17 00:00:00 2001 From: roogue Date: Sun, 8 Jan 2023 18:18:36 +0800 Subject: [PATCH 30/51] Create Manager.ts as for replacement of config --- src/core/Manager.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 src/core/Manager.ts diff --git a/src/core/Manager.ts b/src/core/Manager.ts new file mode 100644 index 0000000..8e74d04 --- /dev/null +++ b/src/core/Manager.ts @@ -0,0 +1,12 @@ +import { existsSync, readFileSync } from "fs"; +import { Collection } from "../struct/Collection"; +import Config from "../struct/Config"; + +const filePath = Config.configFilePath; + +export default class Manager { + protected static collection = new Collection(); + protected static config = existsSync(filePath) + ? new Config(readFileSync(filePath, "utf8")) + : Config.generateConfig(); +} From 6ab899d2a6cdd526196bf9e92c0494ed9b11e469 Mon Sep 17 00:00:00 2001 From: roogue Date: Sun, 8 Jan 2023 18:19:19 +0800 Subject: [PATCH 31/51] Optimize memory usage --- src/core/DownloadManager.ts | 33 +++++++++++---------- src/core/Monitor.ts | 59 ++++++++++++++++++++++++++----------- src/core/OsdbGenerator.ts | 28 +++++++----------- 3 files changed, 69 insertions(+), 51 deletions(-) diff --git a/src/core/DownloadManager.ts b/src/core/DownloadManager.ts index a44864b..0d18015 100644 --- a/src/core/DownloadManager.ts +++ b/src/core/DownloadManager.ts @@ -4,9 +4,8 @@ import _path from "path"; import OcdlError from "../struct/OcdlError"; import Util from "../util"; import EventEmitter from "events"; -import { config } from "../config"; import type { BeatMapSet } from "../struct/BeatMapSet"; -import type { Collection } from "../struct/Collection"; +import Manager from "./Manager"; interface DownloadManagerEvents { downloaded: (beatMapSet: BeatMapSet) => void; @@ -16,7 +15,7 @@ interface DownloadManagerEvents { downloading: (beatMapSet: BeatMapSet) => void; } -export declare interface DownloadManager { +export declare interface DownloadManager extends Manager { on( event: U, listener: DownloadManagerEvents[U] @@ -28,30 +27,32 @@ export declare interface DownloadManager { ): boolean; } -export class DownloadManager extends EventEmitter { +export class DownloadManager extends EventEmitter implements DownloadManager { path: string; parallel: boolean; concurrency: number; osuMirrorUrl: string; altOsuMirrorUrl: string; - collection: Collection; not_downloaded: BeatMapSet[] = []; - constructor(collection: Collection) { + constructor() { super(); - this.path = _path.join(config.directory, collection.name); - this.parallel = config.parallel; - this.concurrency = config.concurrency; - this.osuMirrorUrl = config.osuMirrorApiUrl; - this.altOsuMirrorUrl = config.altOsuMirrorUrl; - this.collection = collection; + + this.path = _path.join( + Manager.config.directory, + Manager.collection.getReplacedName() + ); + this.parallel = Manager.config.parallel; + this.concurrency = Manager.config.concurrency; + this.osuMirrorUrl = Manager.config.osuMirrorApiUrl; + this.altOsuMirrorUrl = Manager.config.altOsuMirrorUrl; } public async bulk_download(): Promise { if (this.parallel) { await this.impulse(); } else { - this.collection.beatMapSets.forEach(async (beatMapSet) => { + Manager.collection.beatMapSets.forEach(async (beatMapSet) => { await this._downloadFile(beatMapSet); }); } @@ -132,9 +133,9 @@ export class DownloadManager extends EventEmitter { } private async impulse(): Promise { - const keys = Array.from(this.collection.beatMapSets.keys()); + const keys = Array.from(Manager.collection.beatMapSets.keys()); const loop_amount = Math.ceil( - this.collection.beatMapSets.size / this.concurrency + Manager.collection.beatMapSets.size / this.concurrency ); for (let i = 0; i < loop_amount; i++) { @@ -146,7 +147,7 @@ export class DownloadManager extends EventEmitter { const range = keys.slice(start, end); for (const id of range) { - const beatMapSet = this.collection.beatMapSets.get(id)!; // always have a value + const beatMapSet = Manager.collection.beatMapSets.get(id)!; // always have a value promises.push(this._downloadFile(beatMapSet)); } diff --git a/src/core/Monitor.ts b/src/core/Monitor.ts index fffc241..1a8108e 100644 --- a/src/core/Monitor.ts +++ b/src/core/Monitor.ts @@ -1,12 +1,12 @@ import chalk from "chalk"; import { log, clear } from "console"; -import { config } from "../config"; -import type { Collection } from "../struct/Collection"; import { Message, Msg } from "../struct/Message"; import OcdlError from "../struct/OcdlError"; +import Util from "../util"; +import Manager from "./Manager"; interface Condition { - mode: string; + new_version: string; retry_input: boolean; retry_mode: boolean; fetched_collection: number; @@ -14,21 +14,19 @@ interface Condition { download_log: string[]; } -export default class Monitor { - private readonly version: string; +export default class Monitor extends Manager { + readonly version: string; private progress = 0; private prompt = require("prompt-sync")({ sigint: true }); private readonly task: Record void>; - readonly collection: Collection; - readonly condition: Condition; - constructor(collection: Collection) { - this.collection = collection; + constructor() { + super(); this.condition = { - mode: config.mode.toString(), + new_version: "", retry_input: false, retry_mode: false, fetched_collection: 0, @@ -50,13 +48,25 @@ export default class Monitor { }; } - update(): void { - clear(); + update(): Monitor { + if (1 != 1) clear(); // Header log(chalk.yellow(`osu-collector-dl v${this.version}`)); + + if (this.condition.new_version) { + log( + chalk.yellow( + new Message(Msg.NEW_VERSION, { + version: this.condition.new_version, + url: `https://github.com/roogue/osu-collector-dl/releases/tag/${this.condition.new_version}`, + }).toString() + ) + ); + } + log( chalk.green( - `Collection: ${this.collection.id} - ${this.collection.name} | Mode: ${this.condition.mode}` + `Collection: ${Manager.collection.id} - ${Manager.collection.name} | Mode: ${Manager.config.mode}` ) ); // Display progress according to current task @@ -65,6 +75,8 @@ export default class Monitor { } catch (e) { throw new OcdlError("MESSAGE_GENERATOR_FAILED", e); } + + return this; } freeze(message: string, isErrored: boolean = false): void { @@ -92,7 +104,14 @@ export default class Monitor { appendLog(log: string): void { this.condition.download_log.splice(0, 0, log); - this.condition.download_log.splice(config.logLength, 1); + this.condition.download_log.splice(Manager.config.logLength, 1); + } + + async checkNewVersion() { + // Check for new version + const newVersion = await Util.checkNewVersion(this.version); + if (!newVersion) return; + this.condition.new_version = newVersion; } // Task 1 @@ -111,7 +130,7 @@ export default class Monitor { // Task 3 private p_fetch_collection(): void { - const beatmaps_length = this.collection.beatMapCount.toString(); + const beatmaps_length = Manager.collection.beatMapCount.toString(); log( new Message(Msg.FETCH_DATA, { @@ -124,14 +143,18 @@ export default class Monitor { // Task 4 private p_create_folder(): void { log( - new Message(Msg.CREATE_FOLDER, { name: this.collection.name }).toString() + new Message(Msg.CREATE_FOLDER, { + name: Manager.collection.name, + }).toString() ); } // Task 5 private p_generate_osdb(): void { log( - new Message(Msg.GENERATE_OSDB, { name: this.collection.name }).toString() + new Message(Msg.GENERATE_OSDB, { + name: Manager.collection.name, + }).toString() ); } @@ -140,7 +163,7 @@ export default class Monitor { log( new Message(Msg.DOWNLOAD_FILE, { amount: this.condition.downloaded_beatmapset.toString(), - total: this.collection.beatMapSets.size.toString(), + total: Manager.collection.beatMapSets.size.toString(), }).toString() ); diff --git a/src/core/OsdbGenerator.ts b/src/core/OsdbGenerator.ts index 9fe8f96..3864e5c 100644 --- a/src/core/OsdbGenerator.ts +++ b/src/core/OsdbGenerator.ts @@ -1,24 +1,20 @@ import { BinaryWriter, File, IFile } from "csbinary"; import { openSync, writeFileSync } from "fs"; -import { config } from "../config"; import _path from "path"; -import type { Collection } from "../struct/Collection"; -import Util from "../util"; +import Manager from "./Manager"; -export default class OsdbGenerator { +export default class OsdbGenerator extends Manager { filePath: string; fileName: string; file: IFile; writer: BinaryWriter; - collection: Collection; - constructor(collection: Collection) { - const collectionName = Util.replaceForbiddenChars(collection.name); - - this.fileName = collectionName + ".osdb"; + constructor() { + super(); + this.fileName = Manager.collection.getReplacedName() + ".osdb"; this.filePath = _path.join( - config.directory, - collectionName, // Folder name + Manager.config.directory, + Manager.collection.getReplacedName(), // Folder name this.fileName ); // Create file @@ -27,8 +23,6 @@ export default class OsdbGenerator { this.file = File(openSync(this.filePath, "w")); // "w" for write this.writer = new BinaryWriter(this.file); - - this.collection = collection; } // * Refer https://github.com/Piotrekol/CollectionManager/blob/master/CollectionManagerDll/Modules/FileIO/FileCollections/OsdbCollectionHandler.cs#L89 @@ -41,18 +35,18 @@ export default class OsdbGenerator { this.writer.writeDouble(this.toOADate(new Date())); // Editor - this.writer.writeString(this.collection.uploader.username); + this.writer.writeString(Manager.collection.uploader.username); // Num of collections this.writer.writeInt32(1); // Always 1 // Name - this.writer.writeString(this.collection.name); + this.writer.writeString(Manager.collection.name); // Beatmap count - this.writer.writeInt32(this.collection.beatMapCount); + this.writer.writeInt32(Manager.collection.beatMapCount); - this.collection.beatMapSets.forEach((beatMapSet, beatMapSetId) => { + Manager.collection.beatMapSets.forEach((beatMapSet, beatMapSetId) => { beatMapSet.beatMaps.forEach((beatmap, beatMapId) => { // BeatmapId this.writer.writeInt32(beatMapId); From 29ab59835301fdaeae3c4ffffbd3114f37264e78 Mon Sep 17 00:00:00 2001 From: roogue Date: Sun, 8 Jan 2023 18:19:27 +0800 Subject: [PATCH 32/51] Update Logger.ts --- src/core/Logger.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/Logger.ts b/src/core/Logger.ts index 27d5766..2e5714c 100644 --- a/src/core/Logger.ts +++ b/src/core/Logger.ts @@ -38,7 +38,7 @@ export default class Logger { if (!existsSync(path)) { writeFileSync( path, - `=== Missing Log ===\n[ Try to download manually ]\n${url}\n` + `=== Missing Beatmap Sets ===\n[ Try to download manually ]\n${url}\n` ); } else { appendFileSync(path, `${url}\n`); From 4daeb90b0c10918a2c7231d96c58937800d0bd0a Mon Sep 17 00:00:00 2001 From: roogue Date: Sun, 8 Jan 2023 18:20:31 +0800 Subject: [PATCH 33/51] Combine function into worker --- src/index.ts | 71 +++------------------------------------------------- 1 file changed, 3 insertions(+), 68 deletions(-) diff --git a/src/index.ts b/src/index.ts index f4f2d03..74d7522 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,80 +1,15 @@ -import Main from "./core/Main"; -import { config } from "./config"; +import Main from "./core/Worker"; import Logger from "./core/Logger"; import OcdlError from "./struct/OcdlError"; -import Monitor from "./core/Monitor"; -import { Message, Msg } from "./struct/Message"; -import { Collection } from "./struct/Collection"; - -const isOnline = async (): Promise => { - return !!(await require("dns") - .promises.resolve("google.com") - .catch(() => {})); -}; // Script Starts Here (async () => { - // Initiate monitor - const collection = new Collection(); - const monitor = new Monitor(collection); - monitor.update(); - - // Check if internet connection is presence - const onlineStatus = await isOnline(); - if (!onlineStatus) - return monitor.freeze(new Message(Msg.NO_CONNECTION).toString(), true); - - let id: number | null = null; - let mode: number | null = null; - - try { - // task 1 - monitor.next(); - - // Get id - while (id === null) { - monitor.update(); - - const result = Number( - monitor.awaitInput(new Message(Msg.INPUT_ID).toString(), "none") - ); - isNaN(result) ? (monitor.condition.retry_input = true) : (id = result); // check if result is valid - } - - monitor.collection.id = id; - - // task 2 - monitor.next(); - - // Get mode - while (mode === null) { - monitor.update(); - const result = String( - monitor.awaitInput( - new Message(Msg.INPUT_MODE, { - mode: config.mode === 2 ? "Yes" : "No", - }).toString(), - config.mode.toString() - ) - ); - if (["n", "no", "1"].includes(result)) mode = 1; - if (["y", "yes", "ass", "2"].includes(result)) mode = 2; - if (mode === null) monitor.condition.retry_mode = true; - } - - monitor.setCondition({ mode: mode.toString() }); - config.mode = mode; - } catch (e) { - Logger.generateErrorLog(new OcdlError("GET_USER_INPUT_FAILED", e)); - return; - } - - const main = new Main(monitor); + const main = new Main(); try { await main.run(); } catch (e) { if (e instanceof OcdlError) Logger.generateErrorLog(e); - monitor.freeze("An error occurred: " + e, true); + main.monitor.freeze("An error occurred: " + e, true); } })(); From 4152d8fb8e52b629d576ee428cee3cc9d0c20732 Mon Sep 17 00:00:00 2001 From: roogue Date: Sun, 8 Jan 2023 18:20:58 +0800 Subject: [PATCH 34/51] Forget to rename hehe --- src/index.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/index.ts b/src/index.ts index 74d7522..ae1eaa0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,15 +1,15 @@ -import Main from "./core/Worker"; +import Worker from "./core/Worker"; import Logger from "./core/Logger"; import OcdlError from "./struct/OcdlError"; // Script Starts Here (async () => { - const main = new Main(); + const worker = new Worker(); try { - await main.run(); + await worker.run(); } catch (e) { if (e instanceof OcdlError) Logger.generateErrorLog(e); - main.monitor.freeze("An error occurred: " + e, true); + worker.monitor.freeze("An error occurred: " + e, true); } })(); From 388ed0fa4e1e07e08cfe9840c89129ebce6fe2c5 Mon Sep 17 00:00:00 2001 From: roogue Date: Sun, 8 Jan 2023 18:21:40 +0800 Subject: [PATCH 35/51] add shorthanded function --- src/struct/BeatMapSet.ts | 5 +++++ src/struct/Collection.ts | 6 ++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/struct/BeatMapSet.ts b/src/struct/BeatMapSet.ts index 5532358..ad371fe 100644 --- a/src/struct/BeatMapSet.ts +++ b/src/struct/BeatMapSet.ts @@ -22,6 +22,11 @@ export class BeatMapSet { this.beatMaps = this._resolveBeatMaps(beatmaps); } + getReplacedName(): string | null { + if (!this.title) return null; + return Util.replaceForbiddenChars(this.title); + } + private _resolveBeatMaps(beatMapJson: BeatMapType[]): Map { return beatMapJson.reduce((acc, current) => { try { diff --git a/src/struct/Collection.ts b/src/struct/Collection.ts index 8cee7b9..2a4710c 100644 --- a/src/struct/Collection.ts +++ b/src/struct/Collection.ts @@ -19,8 +19,6 @@ export class Collection { username: "Unknown", }; - constructor() {} - resolveData(jsonData: Record = {}) { const und = Util.checkUndefined(jsonData, [ "id", @@ -41,6 +39,10 @@ export class Collection { this.beatMapCount = beatmapCount; } + getReplacedName(): string { + return Util.replaceForbiddenChars(this.name); + } + resolveFullData(jsonData: Record[]): void { if (!jsonData.length) throw new OcdlError("CORRUPTED_RESPONSE", "No beatmap found"); From dbc6eb4e76b5317e33cbeab044868c34c3b0df06 Mon Sep 17 00:00:00 2001 From: roogue Date: Sun, 8 Jan 2023 18:21:55 +0800 Subject: [PATCH 36/51] Check for new version before the program runs --- src/struct/Message.ts | 2 ++ src/util.ts | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/src/struct/Message.ts b/src/struct/Message.ts index 452d7f1..0d74492 100644 --- a/src/struct/Message.ts +++ b/src/struct/Message.ts @@ -21,6 +21,8 @@ export class Message { export enum Msg { NO_CONNECTION = "This script only runs with presence of internet connection.", + NEW_VERSION = "New version ({{version}}) is available! Download latest version: {{url}}", + INPUT_ID = "Enter the collection ID you want to download:", INPUT_ID_ERR = "ID should be a number, Ex: '44' (without the quote)", INPUT_MODE = "Generate .osdb file? (y/n) (Default: {{mode}}):", diff --git a/src/util.ts b/src/util.ts index 6f7a244..395ccb4 100644 --- a/src/util.ts +++ b/src/util.ts @@ -1,3 +1,5 @@ +import { request } from "undici"; + export default class Util { static isBoolean(obj: any): boolean { return !!obj === obj; @@ -8,6 +10,44 @@ export default class Util { return str.replace(regex, ""); } + static async isOnline(): Promise { + return !!(await require("dns") + .promises.resolve("google.com") + .catch(() => {})); + } + + static async checkNewVersion( + current_version: string + ): Promise { + if (current_version === "Unknown") return null; + const res = await request( + "https://api.github.com/repos/roogue/osu-collector-dl/releases", + { + method: "GET", + headers: { + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": `osu-collector-dl/v${current_version}`, + }, + query: { + per_page: 1, + }, + } + ).catch(() => null); + + if (!res || res.statusCode !== 200) return null; + const data = (await res.body.json().catch(() => null)) as + | Record[] + | null; + if (!data) return null; + + // Check version + const version = data[0].tag_name as string; + if (version === "v" + current_version) return null; + + return version; + } + static checkUndefined( obj: Record, fields: string[] From 6af52c0260f2bd85a02926c4132d0e949a4c50f3 Mon Sep 17 00:00:00 2001 From: roogue Date: Sun, 8 Jan 2023 18:22:40 +0800 Subject: [PATCH 37/51] Remove debug thingy --- src/core/Monitor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/Monitor.ts b/src/core/Monitor.ts index 1a8108e..bb64840 100644 --- a/src/core/Monitor.ts +++ b/src/core/Monitor.ts @@ -49,7 +49,7 @@ export default class Monitor extends Manager { } update(): Monitor { - if (1 != 1) clear(); + clear(); // Header log(chalk.yellow(`osu-collector-dl v${this.version}`)); From 5582c83bcf088f3df601e70a318560052a9ff3ef Mon Sep 17 00:00:00 2001 From: roogue Date: Sun, 8 Jan 2023 20:31:16 +0800 Subject: [PATCH 38/51] Create Constant.ts --- src/struct/Constant.ts | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 src/struct/Constant.ts diff --git a/src/struct/Constant.ts b/src/struct/Constant.ts new file mode 100644 index 0000000..524fbd9 --- /dev/null +++ b/src/struct/Constant.ts @@ -0,0 +1,7 @@ +export enum Constant { + OsuCollectorApiUrl = "https://osucollector.com/api/collections/", + OsuMirrorApiUrl = "https://api.chimu.moe/v1/download/", + OsuMirrorAltApiUrl = "https://kitsu.moe/api/d/", + GithubReleaseUrl = "https://github.com/roogue/osu-collector-dl/releases/tag/", + GithubReleaseApiUrl = "https://api.github.com/repos/roogue/osu-collector-dl/releases", +} From 7b76682e52f98ee9b52a51579a4980553c4199ec Mon Sep 17 00:00:00 2001 From: roogue Date: Sun, 8 Jan 2023 20:32:36 +0800 Subject: [PATCH 39/51] Improve code commenting --- src/core/DownloadManager.ts | 51 ++++++++++++++++++++------------- src/core/Logger.ts | 20 +++++++++---- src/core/Manager.ts | 6 ++-- src/core/Monitor.ts | 24 +++++++++++----- src/core/OsdbGenerator.ts | 49 +++++++++++++++++-------------- src/core/Worker.ts | 53 +++++++++++++++++++++------------- src/struct/BeatMap.ts | 5 ++-- src/struct/BeatMapSet.ts | 9 ++++-- src/struct/Collection.ts | 15 ++++++++++ src/struct/Config.ts | 57 ++++++++++++++++++------------------- src/struct/Message.ts | 19 +++++++++---- src/struct/OcdlError.ts | 3 ++ src/util.ts | 26 ++++++++--------- 13 files changed, 207 insertions(+), 130 deletions(-) diff --git a/src/core/DownloadManager.ts b/src/core/DownloadManager.ts index 0d18015..a58ba0b 100644 --- a/src/core/DownloadManager.ts +++ b/src/core/DownloadManager.ts @@ -6,10 +6,13 @@ import Util from "../util"; import EventEmitter from "events"; import type { BeatMapSet } from "../struct/BeatMapSet"; import Manager from "./Manager"; +import { Constant } from "../struct/Constant"; +// Define an interface for the events that the DownloadManager class can emit interface DownloadManagerEvents { downloaded: (beatMapSet: BeatMapSet) => void; error: (beatMapSet: BeatMapSet, e: unknown) => void; + // Emitted when all beatmaps have finished downloading (or have failed to download) end: (beatMapSet: BeatMapSet[]) => void; retrying: (beatMapSet: BeatMapSet) => void; downloading: (beatMapSet: BeatMapSet) => void; @@ -29,10 +32,10 @@ export declare interface DownloadManager extends Manager { export class DownloadManager extends EventEmitter implements DownloadManager { path: string; + // Whether to download beatmaps in parallel or sequentially parallel: boolean; + // The number of beatmaps to download in parallel (if `parallel` is true) concurrency: number; - osuMirrorUrl: string; - altOsuMirrorUrl: string; not_downloaded: BeatMapSet[] = []; constructor() { @@ -44,14 +47,15 @@ export class DownloadManager extends EventEmitter implements DownloadManager { ); this.parallel = Manager.config.parallel; this.concurrency = Manager.config.concurrency; - this.osuMirrorUrl = Manager.config.osuMirrorApiUrl; - this.altOsuMirrorUrl = Manager.config.altOsuMirrorUrl; } + // The primary method for downloading beatmaps public async bulk_download(): Promise { + // If `parallel` is true, download beatmaps in parallel using the `_impulse` method if (this.parallel) { - await this.impulse(); + await this._impulse(); } else { + // Otherwise, download beatmaps sequentially Manager.collection.beatMapSets.forEach(async (beatMapSet) => { await this._downloadFile(beatMapSet); }); @@ -60,32 +64,35 @@ export class DownloadManager extends EventEmitter implements DownloadManager { this.emit("end", this.not_downloaded); } + // Downloads a single beatmap file private async _downloadFile( beatMapSet: BeatMapSet, options: { retries: number; alt?: boolean } = { retries: 3 } // Whether or not use the alternative mirror url ): Promise { + // Construct the URL for the beatmap file const url = - (options.alt ? this.altOsuMirrorUrl : this.osuMirrorUrl) + beatMapSet.id; + (options.alt ? Constant.OsuMirrorAltApiUrl : Constant.OsuMirrorApiUrl) + + beatMapSet.id; - // Request download + // Request the download try { this.emit("downloading", beatMapSet); const res = await fetch(url, { method: "GET" }); if (!res.ok) throw `Status code: ${res.status}`; - // Get file name - const fileName = this.getFilename(res); - // Check if directory exists - if (!this.checkIfDirectoryExists()) this.path = process.cwd(); - // Create write stream + // Extract the file name from the response headers + const fileName = this._getFilename(res); + // Check if the specified directory exists + if (!this._checkIfDirectoryExists()) this.path = process.cwd(); + // Create a write stream for the file await new Promise(async (resolve, reject) => { const file = createWriteStream(_path.join(this.path, fileName)); file.on("error", (e) => { reject(e); }); + // Write the file in chunks as the data is received for await (const chunk of res.body!) { - // Write to file file.write(chunk); } @@ -95,28 +102,32 @@ export class DownloadManager extends EventEmitter implements DownloadManager { this.emit("downloaded", beatMapSet); } catch (e) { + // If there are retries remaining, retry the download if (options.retries) { this.emit("retrying", beatMapSet); + // Retry the download with one fewer retry remaining, and use the alternative URL if this is the last retry this._downloadFile(beatMapSet, { alt: options.retries === 1, retries: options.retries - 1, }); } else { + // If there are no retries remaining, emit the "error" event and add the beatmap to the list of failed downloads this.emit("error", beatMapSet, e); this.not_downloaded.push(beatMapSet); } } } - private getFilename(res: Response): string { + private _getFilename(res: Response): string { const headers = res.headers; const contentDisposition = headers.get("content-disposition"); let fileName = "Untitled.osz"; // Default file name - // Extract filename from content-disposition header. + // Extract the file name from the "content-disposition" header if it exists if (contentDisposition) { const result = /filename=([^;]+)/g.exec(contentDisposition); + // If the file name is successfully extracted, decode the string, and replace the forbidden characters if (result) { try { const decoded = decodeURIComponent(result[1]); @@ -132,7 +143,8 @@ export class DownloadManager extends EventEmitter implements DownloadManager { return fileName; } - private async impulse(): Promise { + // Downloads beatmaps in parallel using the `concurrency` property to limit the number of concurrent downloads + private async _impulse(): Promise { const keys = Array.from(Manager.collection.beatMapSets.keys()); const loop_amount = Math.ceil( Manager.collection.beatMapSets.size / this.concurrency @@ -141,13 +153,14 @@ export class DownloadManager extends EventEmitter implements DownloadManager { for (let i = 0; i < loop_amount; i++) { const promises: Promise[] = []; - // Burst + // Calculate the range where the downloads should process const start = i * this.concurrency; const end = (i + 1) * this.concurrency; const range = keys.slice(start, end); + // For the beatmap set on the range, push to a promise and download it in a burst for (const id of range) { - const beatMapSet = Manager.collection.beatMapSets.get(id)!; // always have a value + const beatMapSet = Manager.collection.beatMapSets.get(id)!; // Always have a value promises.push(this._downloadFile(beatMapSet)); } @@ -155,7 +168,7 @@ export class DownloadManager extends EventEmitter implements DownloadManager { } } - private checkIfDirectoryExists(): boolean { + private _checkIfDirectoryExists(): boolean { return existsSync(this.path); } } diff --git a/src/core/Logger.ts b/src/core/Logger.ts index 2e5714c..e4cad34 100644 --- a/src/core/Logger.ts +++ b/src/core/Logger.ts @@ -2,18 +2,23 @@ import type OcdlError from "../struct/OcdlError"; import { existsSync, writeFileSync, appendFileSync } from "fs"; import _path from "path"; +// A utility class for logging errors and missing beatmaps export default class Logger { static readonly errorLogPath = "./ocdl-error.log"; static readonly missingLogPath = "./ocdl-missing.log"; + // Generates an error log file with the given error static generateErrorLog(error: OcdlError): boolean { try { - if (!Logger.checkIfErrorLogFileExists()) { + // Check if the error log file exists + if (!Logger._checkIfErrorLogFileExists()) { + // If it does not, create the file and write the error stack trace to it writeFileSync( Logger.errorLogPath, `=== Error Log ===\n${error.stack}\n=========\n` ); } else { + // If the file does exist, append the error stack trace to it appendFileSync( Logger.errorLogPath, `${error.stack}\n=================\n` @@ -26,16 +31,17 @@ export default class Logger { } } - private static checkIfErrorLogFileExists(): boolean { - return existsSync(Logger.errorLogPath); - } - + // Generates a log file for missing beatmaps static generateMissingLog(folder: string, id: string): boolean { + // Construct the path to the missing beatmaps log file in the given folder const path = _path.join(folder, Logger.missingLogPath); + // Construct the URL for the missing beatmap const url = `https://osu.ppy.sh/beatmapsets/${id}`; try { + // Check if the missing beatmaps log file exists if (!existsSync(path)) { + // If it does not, create the file and write the beatmap URL to it writeFileSync( path, `=== Missing Beatmap Sets ===\n[ Try to download manually ]\n${url}\n` @@ -49,4 +55,8 @@ export default class Logger { return false; } } + + private static _checkIfErrorLogFileExists(): boolean { + return existsSync(Logger.errorLogPath); + } } diff --git a/src/core/Manager.ts b/src/core/Manager.ts index 8e74d04..0b004aa 100644 --- a/src/core/Manager.ts +++ b/src/core/Manager.ts @@ -6,7 +6,7 @@ const filePath = Config.configFilePath; export default class Manager { protected static collection = new Collection(); - protected static config = existsSync(filePath) - ? new Config(readFileSync(filePath, "utf8")) - : Config.generateConfig(); + protected static config = existsSync(filePath) // Check if the path to the config file exist + ? new Config(readFileSync(filePath, "utf8")) // If present, read the config file + : Config.generateConfig(); // If does not present, generate the config file } diff --git a/src/core/Monitor.ts b/src/core/Monitor.ts index bb64840..1052f58 100644 --- a/src/core/Monitor.ts +++ b/src/core/Monitor.ts @@ -1,5 +1,6 @@ import chalk from "chalk"; import { log, clear } from "console"; +import { Constant } from "../struct/Constant"; import { Message, Msg } from "../struct/Message"; import OcdlError from "../struct/OcdlError"; import Util from "../util"; @@ -15,9 +16,13 @@ interface Condition { } export default class Monitor extends Manager { + // Current version of the application readonly version: string; + // Current progress of the application private progress = 0; + // Console prompt for user input and freezing purpose private prompt = require("prompt-sync")({ sigint: true }); + // Object containing functions for each task private readonly task: Record void>; readonly condition: Condition; @@ -39,8 +44,8 @@ export default class Monitor extends Manager { this.task = { 0: () => {}, // Empty function - 1: this.p_input_id.bind(this), // Input id - 2: this.p_input_mode.bind(this), // Input mode + 1: this.p_input_id.bind(this), // Get Input id + 2: this.p_input_mode.bind(this), // Get Input mode 3: this.p_fetch_collection.bind(this), // Fetch collection 4: this.p_create_folder.bind(this), // Fetch collection v2 5: this.p_generate_osdb.bind(this), // Generate osdb @@ -50,25 +55,28 @@ export default class Monitor extends Manager { update(): Monitor { clear(); - // Header + // Header of the console log(chalk.yellow(`osu-collector-dl v${this.version}`)); + // If new version is available, display a message that notice the user if (this.condition.new_version) { log( chalk.yellow( new Message(Msg.NEW_VERSION, { version: this.condition.new_version, - url: `https://github.com/roogue/osu-collector-dl/releases/tag/${this.condition.new_version}`, + url: Constant.GithubReleaseUrl + this.condition.new_version, }).toString() ) ); } + // Display the collection id and name, as well as the current working mode log( chalk.green( `Collection: ${Manager.collection.id} - ${Manager.collection.name} | Mode: ${Manager.config.mode}` ) ); + // Display progress according to current task try { this.task[this.progress](); @@ -80,20 +88,22 @@ export default class Monitor extends Manager { } freeze(message: string, isErrored: boolean = false): void { - // Red color if errored, green if not + // If errored, the message is in red, otherwise green log(isErrored ? chalk.red(message) : chalk.greenBright(message)); // Freeze the console with prompt this.prompt(`Press "Enter" to ${isErrored ? "exit" : "continue"}.`); + // End the whole process if it is errored if (isErrored) process.exit(1); } + // Stop the console and wait for user input awaitInput(message: string, value?: any): string { - return this.prompt(message + " ", value); // Add space + return this.prompt(message + " ", value); } - // Keep progress on track + // To update the progress correspond to the current task next(): void { this.progress++; } diff --git a/src/core/OsdbGenerator.ts b/src/core/OsdbGenerator.ts index 3864e5c..fe964cb 100644 --- a/src/core/OsdbGenerator.ts +++ b/src/core/OsdbGenerator.ts @@ -15,62 +15,66 @@ export default class OsdbGenerator extends Manager { this.filePath = _path.join( Manager.config.directory, Manager.collection.getReplacedName(), // Folder name - this.fileName + this.fileName // File name ); - // Create file + // Create the file writeFileSync(this.filePath, ""); + // Access the file in writing mode this.file = File(openSync(this.filePath, "w")); // "w" for write + // Create a BinaryWriter instance for binary writing this.writer = new BinaryWriter(this.file); } // * Refer https://github.com/Piotrekol/CollectionManager/blob/master/CollectionManagerDll/Modules/FileIO/FileCollections/OsdbCollectionHandler.cs#L89 async writeOsdb(): Promise { try { - // Version 6 does not need to compress + // The version of the osdb file + // Using version o!dm6 so the file does not need to be compressed this.writer.writeString("o!dm6"); - // Date - this.writer.writeDouble(this.toOADate(new Date())); + // OADate + this.writer.writeDouble(this._toOADate(new Date())); - // Editor + // Editor of the collection this.writer.writeString(Manager.collection.uploader.username); - // Num of collections + // Number of collections this.writer.writeInt32(1); // Always 1 - // Name + // Collection name this.writer.writeString(Manager.collection.name); // Beatmap count this.writer.writeInt32(Manager.collection.beatMapCount); + // Write the info for each beatmap in the collection Manager.collection.beatMapSets.forEach((beatMapSet, beatMapSetId) => { beatMapSet.beatMaps.forEach((beatmap, beatMapId) => { - // BeatmapId + // Beatmap id this.writer.writeInt32(beatMapId); - // BeatmapSetId + // Beatmap set id this.writer.writeInt32(beatMapSetId); - // Artist + // Artist of the beatmap set this.writer.writeString(beatMapSet.artist ?? "Unknown"); - // Title + // Title of the beatmap set this.writer.writeString(beatMapSet.title ?? "Unknown"); - // Version + // Version of the beatmap this.writer.writeString(beatmap.version ?? "Unknown"); - // Md5 + // Md5 of the beatmap this.writer.writeString(beatmap.checksum); - // User comment + // User comment, leave it as empty string this.writer.writeString(""); - // Play mode + // The mode of the beatmap this.writer.writeByte(beatmap.mode ?? 0); - // Mod PP Star + // The difficulty rating of the beatmap this.writer.writeDouble(beatmap.difficulty_rating ?? 0); }); }); @@ -79,21 +83,24 @@ export default class OsdbGenerator extends Manager { this.writer.writeInt32(0); // Always 0 // Footer - this.writer.writeString("By Piotrekol"); // Fixed Footer + this.writer.writeString("By Piotrekol"); // Fixed Footer, which is used to determine if the file was corrupted or not } catch (e) { throw e; } finally { - this.closeWriter(); + // Close the writer properly after the writing process was errored or done + this._closeWriter(); } } - private toOADate(date: Date): number { + // Calculation of current date to OADate + private _toOADate(date: Date): number { + // Idk much, the function is copy and pasted from StackOverFlow :) const timezoneOffset = date.getTimezoneOffset() / (60 * 24); const msDateObj = date.getTime() / 86400000 + (25569 - timezoneOffset); return msDateObj; } - private closeWriter(): void { + private _closeWriter(): void { this.writer.close(); } } diff --git a/src/core/Worker.ts b/src/core/Worker.ts index 5291d31..51f9c41 100644 --- a/src/core/Worker.ts +++ b/src/core/Worker.ts @@ -10,6 +10,7 @@ import Logger from "./Logger"; import chalk from "chalk"; import { Message, Msg } from "../struct/Message"; import Manager from "./Manager"; +import { Constant } from "../struct/Constant"; export default class Worker extends Manager { monitor: Monitor; @@ -22,60 +23,69 @@ export default class Worker extends Manager { async run(): Promise { // Check if internet connection is presence const onlineStatus = await Util.isOnline(); + // Stop the process if user is not connected to internet if (!onlineStatus) return this.monitor.freeze( new Message(Msg.NO_CONNECTION).toString(), true ); + // Check for new version of this program await this.monitor.checkNewVersion(); let id: number | null = null; let mode: number | null = null; try { - // task 1 + // Task 1 this.monitor.next(); - // Get id + // Get the collection id from user input while (id === null) { this.monitor.update(); const result = Number( this.monitor.awaitInput(new Message(Msg.INPUT_ID).toString(), "none") ); - if (!isNaN(result)) id = result; // check if result is valid + // Check if result is valid + if (!isNaN(result)) id = result; + // Set retry to true to display the hint if user incorrectly inserted unwanted value this.monitor.condition.retry_input = true; } + // Set collection id after getting input from user Manager.collection.id = id; - // task 2 + // Task 2 this.monitor.next(); - // Get mode + // Get the working mode from user input while (mode === null) { this.monitor.update(); + const result = String( this.monitor.awaitInput( new Message(Msg.INPUT_MODE, { mode: Manager.config.mode === 2 ? "Yes" : "No", }).toString(), - Manager.config.mode.toString() + Manager.config.mode.toString() // Use default working mode from config if the user did not insert any value ) ); + // Validate if the user input is 1 or 2 if (["n", "no", "1"].includes(result)) mode = 1; if (["y", "yes", "ass", "2"].includes(result)) mode = 2; + // Set retry to true to display the hint if user incorrectly inserted unwanted value this.monitor.condition.retry_mode = true; } + // Set the working mode after getting input from user Manager.config.mode = mode; } catch (e) { throw new OcdlError("GET_USER_INPUT_FAILED", e); } // Fetch brief collection info - const responseData = await this.fetchCollection(); + const responseData = await this._fetchCollection(); if (responseData instanceof OcdlError) throw responseData; Manager.collection.resolveData(responseData); @@ -83,14 +93,16 @@ export default class Worker extends Manager { this.monitor.next(); this.monitor.update(); - // Fetch full data if user wants generate osdb file + // Fetch full data if user wants to generate osdb file if (Manager.config.mode === 2) { let hasMorePage: boolean = true; + // The cursor which points to the next page let cursor: number = 0; + // Loop through every beatmaps in the collection while (hasMorePage) { // Request v2 collection - const v2ResponseData = await this.fetchCollection(true, cursor); + const v2ResponseData = await this._fetchCollection(true, cursor); if (v2ResponseData instanceof OcdlError) throw v2ResponseData; try { @@ -106,13 +118,12 @@ export default class Worker extends Manager { // Resolve all required data Manager.collection.resolveFullData(beatmaps); - // Set property hasMorePage = hasMore; cursor = nextPageCursor; + // Update the current condition of monitor to display correct data const fetched_collection = this.monitor.condition.fetched_collection + beatmaps.length; - this.monitor.setCondition({ fetched_collection }); this.monitor.update(); } catch (e) { @@ -125,7 +136,7 @@ export default class Worker extends Manager { this.monitor.next(); this.monitor.update(); - // Create folder + // Create folder for downloading beatmaps and generating osdb file try { responseData.name = Util.replaceForbiddenChars(responseData.name); const path = _path.join(Manager.config.directory, responseData.name); @@ -152,12 +163,13 @@ export default class Worker extends Manager { this.monitor.next(); this.monitor.update(); - // Download beatmapSet - // This is added for people who don't want to download beatmaps + // Set a 3 seconds delay before the download start + // This is added for people who only want to generate osdb file console.log(Msg.PRE_DOWNLOAD); await new Promise((r) => setTimeout(r, 3e3)); try { + // Listen to current download state and log into console const downloadManager = new DownloadManager(); downloadManager.on("downloading", (beatMapSet) => { this.monitor.appendLog( @@ -195,8 +207,10 @@ export default class Worker extends Manager { downloadManager.bulk_download(); + // Create a new promise instance to wait every download process done await new Promise((resolve) => { downloadManager.on("end", (beatMapSet) => { + // For beatmap sets which were failed to download, generate a missing log to notice the user for (let i = 0; i < beatMapSet.length; i++) { Logger.generateMissingLog( Manager.collection.name, @@ -211,17 +225,15 @@ export default class Worker extends Manager { } this.monitor.freeze("\nDownload finished"); - - return; } - private async fetchCollection( + private async _fetchCollection( v2: boolean = false, cursor: number = 0 ): Promise | OcdlError> { - // Check version of collection + // Use different endpoint for different version of api request const url = - Manager.config.osuCollectorApiUrl + + Constant.OsuCollectorApiUrl + Manager.collection.id.toString() + (v2 ? "/beatmapsV2" : ""); @@ -229,9 +241,10 @@ export default class Worker extends Manager { v2 ? { perPage: 100, - cursor, + cursor, // Cursor which point to the next page } : {}; + const data = await request(url, { method: "GET", query }) .then(async (res) => { if (res.statusCode !== 200) throw `Status code: ${res.statusCode}`; diff --git a/src/struct/BeatMap.ts b/src/struct/BeatMap.ts index 129cdd4..c25ba9e 100644 --- a/src/struct/BeatMap.ts +++ b/src/struct/BeatMap.ts @@ -3,16 +3,17 @@ import Util from "../util"; import OcdlError from "./OcdlError"; export class BeatMap { - // compulsory + // Compulsory property id: number; checksum: string; - // nullable + // Nullable property version?: string; mode?: number; difficulty_rating?: number; constructor(jsonData: Record) { + // Check if required fields are present in the JSON response const und = Util.checkUndefined(jsonData, ["id", "checksum"]); if (und) throw new OcdlError("CORRUPTED_RESPONSE", `${und} is required`); diff --git a/src/struct/BeatMapSet.ts b/src/struct/BeatMapSet.ts index ad371fe..ed0888c 100644 --- a/src/struct/BeatMapSet.ts +++ b/src/struct/BeatMapSet.ts @@ -4,29 +4,32 @@ import { BeatMap } from "./BeatMap"; import OcdlError from "./OcdlError"; export class BeatMapSet { - // compulsory + // Compulsory property id: number; beatMaps: Map; - // nullable + // Nullable property title?: string; artist?: string; constructor(jsonData: Record) { + // Check if required fields are present in the JSON response const und = Util.checkUndefined(jsonData, ["id", "beatmaps"]); if (und) throw new OcdlError("CORRUPTED_RESPONSE", `${und} is required`); + // Destructure the JSON data and assign to object properties const { id, beatmaps } = jsonData as BeatMapSetType; - this.id = id; this.beatMaps = this._resolveBeatMaps(beatmaps); } + // Returns the title with forbidden characters replaced, or null if title is not present getReplacedName(): string | null { if (!this.title) return null; return Util.replaceForbiddenChars(this.title); } + // Helper function to create a Map of beatmap IDs to BeatMap objects from JSON data private _resolveBeatMaps(beatMapJson: BeatMapType[]): Map { return beatMapJson.reduce((acc, current) => { try { diff --git a/src/struct/Collection.ts b/src/struct/Collection.ts index 2a4710c..ae4b37f 100644 --- a/src/struct/Collection.ts +++ b/src/struct/Collection.ts @@ -19,7 +19,9 @@ export class Collection { username: "Unknown", }; + // Populates the Collection instance with data from the given jsonData object resolveData(jsonData: Record = {}) { + // Check for required fields in the jsonData object const und = Util.checkUndefined(jsonData, [ "id", "name", @@ -27,6 +29,7 @@ export class Collection { "beatmapsets", "beatmapCount", ]); + // Throw an OcdlError if a required field is not present in the jsonData object if (und) throw new OcdlError("CORRUPTED_RESPONSE", `${und} is required`); const { id, name, uploader, beatmapsets, beatmapCount } = @@ -39,15 +42,20 @@ export class Collection { this.beatMapCount = beatmapCount; } + // Returns a sanitized version of the Collection's name with any forbidden characters replaced getReplacedName(): string { return Util.replaceForbiddenChars(this.name); } + // Populates the beatMapSet and beatMap instances within the Collection with data from the given jsonData array resolveFullData(jsonData: Record[]): void { + // Throw an OcdlError if the jsonData array is empty if (!jsonData.length) throw new OcdlError("CORRUPTED_RESPONSE", "No beatmap found"); + // Iterate through each element in the jsonData array for (let i = 0; i < jsonData.length; i++) { + // Check for required fields in the current element of the jsonData array const und = Util.checkUndefined(jsonData[i], [ "id", "mode", @@ -55,6 +63,7 @@ export class Collection { "version", "beatmapset", ]); + // Throw an OcdlError if a required field is not present in the current element of the jsonData array if (und) throw new OcdlError("CORRUPTED_RESPONSE", `${und} is required`); const { id, mode, difficulty_rating, version, beatmapset } = jsonData[ @@ -62,6 +71,7 @@ export class Collection { ] as FullBeatMapType; const beatMapSet = this.beatMapSets.get(beatmapset.id); + // Continue to the next iteration if the BeatMapSet instance was not found if (!beatMapSet) continue; const { title, artist } = beatmapset; @@ -70,20 +80,25 @@ export class Collection { beatMapSet.artist = artist; const beatMap = beatMapSet.beatMaps.get(id); + // Continue to the next iteration if the beatMap instance was not found if (!beatMap) continue; beatMap.difficulty_rating = difficulty_rating; + // Convert the mode field from a string to a number using the ModeByte object beatMap.mode = +ModeByte[mode]; beatMap.version = version; } } + // Returns a Map of beatmap set id to BeatMapSet instance, constructed from the given beatMapSetJson array private _resolveBeatMapSets( beatMapSetJson: BeatMapSetType[] ): Map { + // Reduce the beatMapSetJson array to a Map, adding a new entry for each element in the array return beatMapSetJson.reduce((acc, current) => { try { const map = new BeatMapSet(current); + // Add an entry to the Map with the id of the BeatMapSet instance as the key and the instance as the value acc.set(map.id, map); return acc; } catch (e) { diff --git a/src/struct/Config.ts b/src/struct/Config.ts index 9d1ac37..2c7e32e 100644 --- a/src/struct/Config.ts +++ b/src/struct/Config.ts @@ -5,65 +5,57 @@ import Util from "../util"; import OcdlError from "./OcdlError"; export default class Config { + // Whether the download process should be done in parallel parallel: boolean; - osuCollectorApiUrl: string; - osuMirrorApiUrl: string; - altOsuMirrorUrl: string; + // The number of URLs that should be downloaded in parallel at once concurrency: number; + // The directory to save beatmaps directory: string; + // The mode of operation + // 1: Download BeatmapSet + // 2: Download BeatmapSet + Generate .osdb mode: number; + // The length of the log when downloading beatmapsets logLength: number; + // The path to the config file static readonly configFilePath = "./config.json"; + // Constructs a new Config object from a string of JSON data + // If no data is provided, default values are used constructor(contents?: string) { let config: Record = {}; if (contents) { try { + // Parse the JSON data and store it in the 'config' object config = JSON.parse(contents); } catch (e) { + // If there is an error parsing the JSON data, throw an OcdlError throw Logger.generateErrorLog(new OcdlError("INVALID_CONFIG", e)); } } - // Osucollector's base url - this.osuCollectorApiUrl = "https://osucollector.com/api/collections/"; - - // Osumirror's api url for download beatmap - this.osuMirrorApiUrl = "https://api.chimu.moe/v1/download/"; - - // alt Osu mirror url - this.altOsuMirrorUrl = "https://kitsu.moe/api/d/"; - - // The length of log when downloading beatmapsets + // Set default values for properties if not provided in 'config' object this.logLength = !isNaN(Number(config.logSize)) ? Number(config.logSize) : 15; - - // Whether download process should be done in parallel this.parallel = Util.isBoolean(config.parallel) ? config.parallel : true; - - // How many urls should be downloaded in parallel at once this.concurrency = !isNaN(Number(config.concurrency)) ? Number(config.concurrency) : 10; - - // Directory to save beatmaps - this.directory = this.getPath(config.directory); - - // Mode - // 1: Download BeatmapSet - // 2: Download BeatmapSet + Generate .osdb - this.mode = this.getMode(config.mode); + this.directory = this._getPath(config.directory); + this.mode = this._getMode(config.mode); } + // Generates a default config file if one does not already exist static generateConfig(): Config { - if (!Config.checkIfConfigFileExist()) { + if (!Config._checkIfConfigFileExist()) { writeFileSync( Config.configFilePath, JSON.stringify({ parallel: true, concurrency: 5, - directory: process.cwd(), + logSize: 15, + directory: "", mode: 1, }) ); @@ -71,15 +63,20 @@ export default class Config { return new Config(); } - private static checkIfConfigFileExist(): boolean { + // Check if the config file exists + private static _checkIfConfigFileExist(): boolean { return existsSync(Config.configFilePath); } - private getMode(data: any): number { + // Returns the mode of operation based on the provided data + // If the provided data is invalid, returns 1 (Download BeatmapSet) + private _getMode(data: any): number { return data == 1 ? 1 : data == 2 ? 2 : 1; } - private getPath(data: any): string { + // Returns the directory path based on the provided data + // If the provided data is invalid, returns the current working directory + private _getPath(data: any): string { if (typeof data !== "string") return process.cwd(); return path.isAbsolute(data) ? data : process.cwd(); } diff --git a/src/struct/Message.ts b/src/struct/Message.ts index 0d74492..09e3291 100644 --- a/src/struct/Message.ts +++ b/src/struct/Message.ts @@ -1,23 +1,30 @@ export class Message { + // Message object that will be constructed with a Msg enum value + // and an optional object with variables to be replaced in the message string private message: Msg; private variable: Record; + // Constructor to create a new Message object constructor(message: Msg, variable?: Record) { + // Assign the provided message and variable to the class properties this.message = message; - this.variable = variable ?? {}; + this.variable = variable ?? {}; // default to an empty object if no variable provided } - toString() { + // Method to convert the message to a string with variables replaced + toString(): string { // Replace value if variable is provided - let msg: string = this.message; + let msg: string = this.message; // get the message from the class property for (const [key, value] of Object.entries(this.variable)) { - const regex = new RegExp(`{{${key}}}`, "g"); - msg = msg.replace(regex, value); + // iterate over the variables and replace the placeholders in the message string + const regex = new RegExp(`{{${key}}}`, "g"); // create a regex to match the placeholder + msg = msg.replace(regex, value); // replace the placeholder with the value } - return msg; + return msg; // return the modified message string } } +// Enum with string values representing different messages export enum Msg { NO_CONNECTION = "This script only runs with presence of internet connection.", diff --git a/src/struct/OcdlError.ts b/src/struct/OcdlError.ts index 065038d..83fc06a 100644 --- a/src/struct/OcdlError.ts +++ b/src/struct/OcdlError.ts @@ -1,3 +1,4 @@ +// Types of error in enumerator export enum ErrorType { "INVALID_CONFIG" = "The config is invalid json type", "GET_USER_INPUT_FAILED" = "Error occurred while getting user input", @@ -11,12 +12,14 @@ export enum ErrorType { "CORRUPTED_RESPONSE" = "The api response is corrupted", } +// Returns a string containing the current date, a label, the string value associated with the errorType, and the error itself const getMessage = (type: keyof typeof ErrorType, error: any): string => { return `${new Date()} | [OcdlError]: ${type} - ${ErrorType[type]}\n${error}`; }; export default class OcdlError extends Error { constructor(errorType: keyof typeof ErrorType, error: any) { + // Calls the parent class' constructor and sets the message property of the OcdlError instance super(getMessage(errorType, error)); } } diff --git a/src/util.ts b/src/util.ts index 395ccb4..b07b2e6 100644 --- a/src/util.ts +++ b/src/util.ts @@ -1,4 +1,5 @@ import { request } from "undici"; +import { Constant } from "./struct/Constant"; export default class Util { static isBoolean(obj: any): boolean { @@ -20,20 +21,17 @@ export default class Util { current_version: string ): Promise { if (current_version === "Unknown") return null; - const res = await request( - "https://api.github.com/repos/roogue/osu-collector-dl/releases", - { - method: "GET", - headers: { - Accept: "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28", - "User-Agent": `osu-collector-dl/v${current_version}`, - }, - query: { - per_page: 1, - }, - } - ).catch(() => null); + const res = await request(Constant.GithubReleaseApiUrl, { + method: "GET", + headers: { + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": `osu-collector-dl/v${current_version}`, + }, + query: { + per_page: 1, + }, + }).catch(() => null); if (!res || res.statusCode !== 200) return null; const data = (await res.body.json().catch(() => null)) as From c860a0f0e3a4ecc4f67162420950f0251dfc1782 Mon Sep 17 00:00:00 2001 From: roogue Date: Sun, 8 Jan 2023 20:36:34 +0800 Subject: [PATCH 40/51] Change property name --- src/core/Monitor.ts | 2 +- src/struct/Config.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/core/Monitor.ts b/src/core/Monitor.ts index 1052f58..529a8ce 100644 --- a/src/core/Monitor.ts +++ b/src/core/Monitor.ts @@ -114,7 +114,7 @@ export default class Monitor extends Manager { appendLog(log: string): void { this.condition.download_log.splice(0, 0, log); - this.condition.download_log.splice(Manager.config.logLength, 1); + this.condition.download_log.splice(Manager.config.logSize, 1); } async checkNewVersion() { diff --git a/src/struct/Config.ts b/src/struct/Config.ts index 2c7e32e..c0c0426 100644 --- a/src/struct/Config.ts +++ b/src/struct/Config.ts @@ -16,7 +16,7 @@ export default class Config { // 2: Download BeatmapSet + Generate .osdb mode: number; // The length of the log when downloading beatmapsets - logLength: number; + logSize: number; // The path to the config file static readonly configFilePath = "./config.json"; @@ -35,7 +35,7 @@ export default class Config { } // Set default values for properties if not provided in 'config' object - this.logLength = !isNaN(Number(config.logSize)) + this.logSize = !isNaN(Number(config.logSize)) ? Number(config.logSize) : 15; this.parallel = Util.isBoolean(config.parallel) ? config.parallel : true; From 9280e6d0eb48bebf753a93b47c9cd038fe4f27f3 Mon Sep 17 00:00:00 2001 From: roogue Date: Sun, 8 Jan 2023 20:44:54 +0800 Subject: [PATCH 41/51] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 5197677..97c7c5b 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,8 @@ Below is the data stored in config.json, along with explanations of each setting - `concurrency`: The number of downloads to request at a time. It is recommended to set this to a low number (such as 5) to prevent abuse of the osu!mirror API and potential IP bans or rate limits. +- `logSize`: The number that determines the maximum number of log messages during the download process. + - `directory`: The path to the folder where you want to save the downloaded beatmaps. If no value is provided, the current working directory will be used. Remember to include double quotes around the path! - `mode`: The mode in which the program should operate. Set to `1` to only download the beatmap sets, or `2` to also generate a .osdb file during the download process. You can also specify the mode at the terminal. From 3a9e513a166bb1fe46eac7bf2006b7d67e6fd829 Mon Sep 17 00:00:00 2001 From: roogue Date: Sun, 8 Jan 2023 20:51:41 +0800 Subject: [PATCH 42/51] Added terminal title --- src/core/Monitor.ts | 6 +++--- src/util.ts | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/core/Monitor.ts b/src/core/Monitor.ts index 529a8ce..5f70b41 100644 --- a/src/core/Monitor.ts +++ b/src/core/Monitor.ts @@ -42,6 +42,9 @@ export default class Monitor extends Manager { this.version = (require("../../package.json")?.version ?? "Unknown") as string; // Get current version from package.json + // Set terminal title according to it's version + Util.setTerminalTitle(`osu-collector-dl ${this.version}`); + this.task = { 0: () => {}, // Empty function 1: this.p_input_id.bind(this), // Get Input id @@ -55,9 +58,6 @@ export default class Monitor extends Manager { update(): Monitor { clear(); - // Header of the console - log(chalk.yellow(`osu-collector-dl v${this.version}`)); - // If new version is available, display a message that notice the user if (this.condition.new_version) { log( diff --git a/src/util.ts b/src/util.ts index b07b2e6..3aa7a6f 100644 --- a/src/util.ts +++ b/src/util.ts @@ -57,4 +57,10 @@ export default class Util { } return null; } + + static setTerminalTitle(title: string) { + process.stdout.write( + String.fromCharCode(27) + "]0;" + title + String.fromCharCode(7) + ); + } } From ff92f94a4067fa9de021a3599b2f930f7e64d9de Mon Sep 17 00:00:00 2001 From: roogue Date: Sun, 8 Jan 2023 20:52:51 +0800 Subject: [PATCH 43/51] Update package.json --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index f73fb30..f5c0200 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,8 @@ "build-app": "yarn build-app-win && yarn build-app-linux", "cp-build-win": "cp ./{LICENSE,README.md} ./build/win-x64/ && cp ./config.json.example ./build/win-x64/config.json", "cp-build-linux": "cp ./{LICENSE,README.md} ./build/linux-arm64/ && cp ./config.json.example ./build/linux-arm64/config.json", - "cp-build": "yarn cp-build-win && yarn cp-build-linux" + "cp-build": "yarn cp-build-win && yarn cp-build-linux", + "pack-app": "yarn build-app & yarn cp-build" }, "license": "MIT", "dependencies": { From d6898d766151ae66213effef4dd47a6c1673055e Mon Sep 17 00:00:00 2001 From: roogue Date: Sun, 8 Jan 2023 20:55:26 +0800 Subject: [PATCH 44/51] Update package.json --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index f5c0200..28f1ec8 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,8 @@ "cp-build-win": "cp ./{LICENSE,README.md} ./build/win-x64/ && cp ./config.json.example ./build/win-x64/config.json", "cp-build-linux": "cp ./{LICENSE,README.md} ./build/linux-arm64/ && cp ./config.json.example ./build/linux-arm64/config.json", "cp-build": "yarn cp-build-win && yarn cp-build-linux", - "pack-app": "yarn build-app & yarn cp-build" + "clean-folder": "rm -r ./build", + "pack-app": "clean-folder && yarn build-app & yarn cp-build" }, "license": "MIT", "dependencies": { From b978f1c316e17677a7227b75f9e512a8f4fc6b16 Mon Sep 17 00:00:00 2001 From: roogue Date: Sun, 8 Jan 2023 21:02:30 +0800 Subject: [PATCH 45/51] Update package.json --- package.json | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 28f1ec8..6fd3837 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "3.0.0", "main": "./dist/index.js", "scripts": { - "build": "yarn run build-app && yarn run cp-build", + "build": "yarn run clean-folder && yarn run build-app && yarn run cp-build", "start": "yarn build-ts && node .", "build-ts": "tsc -p .", "build-app-win": "yarn build-ts && pkg -t node16-win-x64 -o ./build/win-x64/osu-collector-dl.exe ./dist/index.js --public", @@ -12,8 +12,7 @@ "cp-build-win": "cp ./{LICENSE,README.md} ./build/win-x64/ && cp ./config.json.example ./build/win-x64/config.json", "cp-build-linux": "cp ./{LICENSE,README.md} ./build/linux-arm64/ && cp ./config.json.example ./build/linux-arm64/config.json", "cp-build": "yarn cp-build-win && yarn cp-build-linux", - "clean-folder": "rm -r ./build", - "pack-app": "clean-folder && yarn build-app & yarn cp-build" + "clean-folder": "rm -r ./build" }, "license": "MIT", "dependencies": { From 4773270282eb220d16a3b7d64a40140d2e70cabd Mon Sep 17 00:00:00 2001 From: roogue Date: Sun, 8 Jan 2023 21:26:57 +0800 Subject: [PATCH 46/51] Update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6fd3837..7d63f05 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "osu-collector-dl", - "version": "3.0.0", + "version": "2.7.0", "main": "./dist/index.js", "scripts": { "build": "yarn run clean-folder && yarn run build-app && yarn run cp-build", From 07d4e129ebfdcc9e8430a742793c301e338f6ce9 Mon Sep 17 00:00:00 2001 From: roogue Date: Mon, 9 Jan 2023 14:04:42 +0800 Subject: [PATCH 47/51] Add more FAQ --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 97c7c5b..d757f41 100644 --- a/README.md +++ b/README.md @@ -41,17 +41,29 @@ Below is the data stored in config.json, along with explanations of each setting ## FAQ ### It says "Retrying" during the download process, am I doing anything wrong? + > No, you are not doing anything wrong. It is normal for API requests to sometimes fail due to factors such as rate limiting and internet connection issues. The script has a built-in retrying process that will handle these issues automatically. It is expected to see the "Retrying" message during the download process. ### I want the beatmaps to be automatically added to my collections. Is that possible? + > Unfortunately, this feature will not be implemented as directly modifying your personal osu! folder is risky and could potentially result in corrupted files. It is recommended to use [Collection Manager](https://github.com/Piotrekol/CollectionManager) (CM) by Piotrekol to modify your collection for more stable functionality. ### Why won't my program even start? The program shuts off right after I opened it. + > There could be several reasons why your program is not starting. One potential cause is that you have incorrectly edited the config.json file, such as forgetting to include double quotes around the directory path. If you are not sure what the problem is, try reinstalling the program to see if that resolves the issue. +### The program freezes in the middle of the process without displaying any error messages. What can I do? + +> One possible solution is to try pressing Enter on your keyboard to see if that prompts the program to continue. This can sometimes happen if you accidentally clicked on the terminal window, which can cause the program to pause. + +### I accidentally downloaded the wrong collection. How can I stop the downloads? + +> To stop the downloads, you can simply close the terminal window. This will terminate the program. Alternatively, you can try pressing CTRL+C on your keyboard, which will send a signal to the program to stop running. + ### I have tried following the FAQ above, but it didn't solve my problem. The problem I am experiencing is not listed in the FAQ. + > If you are experiencing a problem that is not covered in the FAQ and you need assistance, it is welcome to open an issue on the [Issue Page](https://github.com/roogue/osu-collector-dl/issues). After navigating to the issue page, click the green "New issue" button on the page and follow the instructions to describe your problem in as much detail as possible. This will allow the maintainers of the project to better understand and help troubleshoot the issue you are experiencing. ## License -This project is licensed under the MIT License. See the [LICENSE](https://choosealicense.com/licenses/mit/) file for details. \ No newline at end of file +This project is licensed under the MIT License. See the [LICENSE](https://choosealicense.com/licenses/mit/) file for details. From 68880c5446b708aead8f14043700955185e6532b Mon Sep 17 00:00:00 2001 From: roogue Date: Mon, 9 Jan 2023 14:04:58 +0800 Subject: [PATCH 48/51] Fix version not displayed correctly --- src/core/Monitor.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/Monitor.ts b/src/core/Monitor.ts index 5f70b41..2d79a3a 100644 --- a/src/core/Monitor.ts +++ b/src/core/Monitor.ts @@ -43,7 +43,7 @@ export default class Monitor extends Manager { "Unknown") as string; // Get current version from package.json // Set terminal title according to it's version - Util.setTerminalTitle(`osu-collector-dl ${this.version}`); + Util.setTerminalTitle(`osu-collector-dl v${this.version}`); this.task = { 0: () => {}, // Empty function From 02b6ddf14b0ee1ebc8a9c29876058e85459d954f Mon Sep 17 00:00:00 2001 From: roogue Date: Mon, 9 Jan 2023 14:05:52 +0800 Subject: [PATCH 49/51] Use asynchronous version checker --- src/core/Worker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/Worker.ts b/src/core/Worker.ts index 51f9c41..b8b598d 100644 --- a/src/core/Worker.ts +++ b/src/core/Worker.ts @@ -31,7 +31,7 @@ export default class Worker extends Manager { ); // Check for new version of this program - await this.monitor.checkNewVersion(); + this.monitor.checkNewVersion(); let id: number | null = null; let mode: number | null = null; From 985ae28d68d0f3a6a5185a9735732635bb221749 Mon Sep 17 00:00:00 2001 From: roogue Date: Mon, 9 Jan 2023 14:06:28 +0800 Subject: [PATCH 50/51] Fixed minor bug where the beatmapCount is not accurate --- src/struct/Collection.ts | 15 +++++++++++---- src/types.ts | 1 - 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/struct/Collection.ts b/src/struct/Collection.ts index ae4b37f..d8c013e 100644 --- a/src/struct/Collection.ts +++ b/src/struct/Collection.ts @@ -27,19 +27,17 @@ export class Collection { "name", "uploader", "beatmapsets", - "beatmapCount", ]); // Throw an OcdlError if a required field is not present in the jsonData object if (und) throw new OcdlError("CORRUPTED_RESPONSE", `${und} is required`); - const { id, name, uploader, beatmapsets, beatmapCount } = - jsonData as CollectionType; + const { id, name, uploader, beatmapsets } = jsonData as CollectionType; this.id = id; this.name = name; this.uploader = uploader; this.beatMapSets = this._resolveBeatMapSets(beatmapsets); - this.beatMapCount = beatmapCount; + this.beatMapCount = this._getBeatMapCount(beatmapsets); } // Returns a sanitized version of the Collection's name with any forbidden characters replaced @@ -106,4 +104,13 @@ export class Collection { } }, new Map()); } + + // Reduce the beatMapSetJson array to the number of beatmaps. + // This alternative method is used because the response from osu!Collector API is not always accurate + private _getBeatMapCount(beatMapSetJson: BeatMapSetType[]): number { + return beatMapSetJson.reduce((acc, current) => { + const length = current.beatmaps.length; + return acc + length; + }, 0); + } } diff --git a/src/types.ts b/src/types.ts index 11a060a..d4c890e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -12,7 +12,6 @@ export interface BeatMapSetType { export interface CollectionType { beatmapIds: BeatMapType[]; beatmapsets: BeatMapSetType[]; - beatmapCount: number; id: number; name: string; uploader: { From 7d3d6a3f911d86da74073f022b5d2892d49815db Mon Sep 17 00:00:00 2001 From: roogue Date: Mon, 9 Jan 2023 14:44:21 +0800 Subject: [PATCH 51/51] Fix sequential download not working as expected --- src/core/DownloadManager.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/DownloadManager.ts b/src/core/DownloadManager.ts index a58ba0b..d1fbd47 100644 --- a/src/core/DownloadManager.ts +++ b/src/core/DownloadManager.ts @@ -56,9 +56,9 @@ export class DownloadManager extends EventEmitter implements DownloadManager { await this._impulse(); } else { // Otherwise, download beatmaps sequentially - Manager.collection.beatMapSets.forEach(async (beatMapSet) => { + for (const [_, beatMapSet] of Manager.collection.beatMapSets) { await this._downloadFile(beatMapSet); - }); + } } this.emit("end", this.not_downloaded);