feat: separate puppeteer and puppeteer-core (#9023)

This PR moves the puppeteer source code into separate mono-repo packages: - `puppeteer` and `puppeteer-core` are now separated into their own packages. - `puppeteer-core` has a new exports called `puppeteer-core/internal` for internal usage. Tests and various tools have been updated to accommodate the migration.
puppeteer · Oct 5, 2022 · f42336c · f42336c
1 parent 3aee641
commit f42336c
Show file tree

Hide file tree

Showing 193 changed files with 6,752 additions and 15,278 deletions.
diff --git a/.eslintignore b/.eslintignore
@@ -8,8 +8,8 @@ lib/
 
 # Generated files
 **/*.tsbuildinfo
-puppeteer.api.json
-puppeteer*.tgz
+*.api.json
+*.tgz
 yarn.lock
 .docusaurus/
 .cache-loader
@@ -18,7 +18,9 @@ yarn.lock
 test/output-*/
 .dev_profile*
 coverage/
-src/generated
+generated/
+.eslintcache
+/.cache/
 
 # IDE Artifacts
 .vscode
@@ -35,6 +37,8 @@ npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
 
+# Wireit
+.wireit
 ## [END] Keep in sync with .gitignore
 
 # ESLint ignores.

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -60,7 +60,7 @@ jobs:
       - name: Setup cache for Chromium binary
         uses: actions/cache@v3
         with:
-          path: .local-chromium
+          path: packages/puppeteer-core/.local-chromium
           key: ${{ runner.os }}-chromium-${{ hashFiles('src/revisions.ts') }}
       - name: Install dependencies
         run: npm ci
@@ -226,23 +226,25 @@ jobs:
       - name: Setup cache for Firefox binary
         uses: actions/cache@v3
         with:
-          path: .local-firefox
+          path: packages/puppeteer-core/.local-firefox
           key: ${{ runner.os }}-firefox-${{ hashFiles('src/revisions.ts') }}
       - name: Set up Node.js
         uses: actions/setup-node@v3.4.1
         with:
           node-version: ${{ matrix.spec.node }}
+      - name: Update NPM
+        run: npm install -g npm@latest
       - name: Install dependencies with Chromium
-        run: npm install
+        run: npm ci
       - name: Install Firefox
         env:
           PUPPETEER_PRODUCT: firefox
-        run: npm install
+        run: npm ci
       - name: Install linux dependencies.
         if: ${{ matrix.spec.xvfb }}
         run: sudo apt-get install xvfb
       - name: Build
-        run: npm run build:dev
+        run: npm run build:test
       - name: Test types
         run: npm run test:types
       - name: Run all tests with xvfb
@@ -278,7 +280,7 @@ jobs:
       - name: Install dependencies
         run: |
           npm install
-          ls .local-chromium
+          ls packages/puppeteer-core/.local-chromium
       - name: Build
         run: |
           npm run build

diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml
@@ -27,7 +27,7 @@ jobs:
         working-directory: ./website
         run: |
           npm install
-          npm run docusaurus docs:version $(jq -r .version ../package.json)
+          npm run docusaurus docs:version $(jq -r .version ../packages/puppeteer/package.json)
           npm run archive
       - name: Re-build docs after versioning
         env:

diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -16,24 +16,12 @@ jobs:
         uses: actions/checkout@v3
       - name: Install dependencies
         run: npm install
-      - name: Build puppeteer
+      - name: Build packages
         run: npm run build
-      - name: Publish puppeteer
+      - name: Publish packages
         env:
-          NPM_TOKEN: ${{secrets.NPM_TOKEN_PUPPETEER}}
+          NODE_AUTH_TOKEN: ${{secrets.NPM_TOKEN_RELEASE}}
         run: |
           npm config set registry 'https://wombat-dressing-room.appspot.com/'
-          npm config set '//wombat-dressing-room.appspot.com/:_authToken' '${NPM_TOKEN}'
-          echo "Publishing puppeteer"
-          npm publish
-          # DEPRECATED_RANGE=$(node utils/get_deprecated_version_range.js)
-          # echo "Deprecating old puppeteer versions: $DEPRECATED_RANGE"
-          # npm deprecate puppeteer@$DEPRECATED_RANGE "Version no longer supported. Upgrade to @latest"
-      - name: Publish puppeteer-core
-        env:
-          NPM_TOKEN: ${{secrets.NPM_TOKEN_PUPPETEER_CORE}}
-        run: |
-          utils/prepare_puppeteer_core.js
-          npm config set registry 'https://wombat-dressing-room.appspot.com/'
-          npm config set '//wombat-dressing-room.appspot.com/:_authToken' '${NPM_TOKEN}'
-          npm publish
+          npm config set '//wombat-dressing-room.appspot.com/:_authToken' $NODE_AUTH_TOKEN
+          npm publish --workspaces
diff --git a/.github/workflows/tot-ci.yml b/.github/workflows/tot-ci.yml
@@ -43,7 +43,7 @@ jobs:
           echo "Installing revision $REV"
           cat src/revisions.ts | sed "s/[0-9]\{6,\}/$REV/" > src/revisions.ts.replaced
           mv src/revisions.ts.replaced src/revisions.ts
-          npm run build:dev
+          npm run build:test
           npm install
 
       - name: Run unit tests in headless
@@ -81,7 +81,7 @@ jobs:
           echo "Installing revision $REV"
           cat src/revisions.ts | sed "s/[0-9]\{6,\}/$REV/" > src/revisions.ts.replaced
           mv src/revisions.ts.replaced src/revisions.ts
-          npm run build:dev
+          npm run build:test
           npm install
 
       - name: Run unit tests in headful
@@ -119,7 +119,7 @@ jobs:
           echo "Installing revision $REV"
           cat src/revisions.ts | sed "s/[0-9]\{6,\}/$REV/" > src/revisions.ts.replaced
           mv src/revisions.ts.replaced src/revisions.ts
-          npm run build:dev
+          npm run build:test
           npm install
 
       - name: Run unit tests in chrome headless

diff --git a/.gitignore b/.gitignore
@@ -7,8 +7,8 @@ lib/
 
 # Generated files
 **/*.tsbuildinfo
-puppeteer.api.json
-puppeteer*.tgz
+*.api.json
+*.tgz
 yarn.lock
 .docusaurus/
 .cache-loader
@@ -17,7 +17,9 @@ yarn.lock
 test/output-*/
 .dev_profile*
 coverage/
-src/generated
+generated/
+.eslintcache
+/.cache/
 
 # IDE Artifacts
 .vscode
@@ -33,3 +35,6 @@ src/generated
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
+
+# Wireit
+.wireit
diff --git a/.prettierignore b/.prettierignore
@@ -8,8 +8,8 @@ lib/
 
 # Generated files
 **/*.tsbuildinfo
-puppeteer.api.json
-puppeteer*.tgz
+*.api.json
+*.tgz
 yarn.lock
 .docusaurus/
 .cache-loader
@@ -18,7 +18,9 @@ yarn.lock
 test/output-*/
 .dev_profile*
 coverage/
-src/generated
+generated/
+.eslintcache
+/.cache/
 
 # IDE Artifacts
 .vscode
@@ -34,13 +36,15 @@ src/generated
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
+
+# Wireit
+.wireit
 ## [END] Keep in sync with .gitignore
 
 # Prettier-only ignores.
 assets/
 CHANGELOG.md
 package-lock.json
-package.json
 test/assets/
 docs/
 versioned_*/
diff --git a/.release-please-manifest.json b/.release-please-manifest.json
@@ -1,3 +1,5 @@
 {
-  ".": "18.1.0"
+  "puppeteer": "18.1.0",
+  "puppeteer-core": "18.1.0",
+  "testserver": "0.5.0"
 }
diff --git a/compat/cjs/tsconfig.json b/compat/cjs/tsconfig.json
diff --git a/compat/esm/tsconfig.json b/compat/esm/tsconfig.json
diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -15,10 +15,11 @@ RUN apt-get update \
 WORKDIR /home/pptruser
 
 COPY puppeteer-latest.tgz /home/pptruser/puppeteer-latest.tgz
+COPY puppeteer-core-latest.tgz /home/pptruser/puppeteer-core-latest.tgz
 
-# Install puppeteer into /home/pptruser/node_modules.
-RUN npm i ./puppeteer-latest.tgz \
-    && rm puppeteer-latest.tgz \
+# Install puppeteer and puppeteer-core into /home/pptruser/node_modules.
+RUN npm i ./puppeteer-core-latest.tgz ./puppeteer-latest.tgz \
+    && rm ./puppeteer-core-latest.tgz ./puppeteer-latest.tgz \
     # Add user so we don't need --no-sandbox.
     # same layer as npm install to keep re-chowned files from using up several hundred MBs more space
     && groupadd -r pptruser && useradd -r -g pptruser -G audio,video pptruser \

diff --git a/docker/pack.sh b/docker/pack.sh
@@ -4,10 +4,13 @@
 # Expected cwd: project root directory.
 
 set -e
-set +x
 
-FILENAME=$(npm pack)
+cd docker
 
-echo $FILENAME
+npm pack --workspace puppeteer --workspace puppeteer-core --pack-destination .
 
-mv $FILENAME docker/puppeteer-latest.tgz
+rm -f puppeteer-core-latest.tgz
+rm -f puppeteer-latest.tgz
+
+mv puppeteer-core-*.tgz puppeteer-core-latest.tgz
+mv puppeteer-[0-9]*.tgz puppeteer-latest.tgz
diff --git a/docs/api/puppeteer.connect.md b/docs/api/puppeteer.connect.md
@@ -7,6 +7,6 @@ sidebar_label: connect
 **Signature:**
 
 ```typescript
-connect: (options: import('./types.js').ConnectOptions) =>
-  Promise<import('./types.js').Browser>;
+connect: (options: import('./types').ConnectOptions) =>
+  Promise<import('./types').Browser>;
 ```
diff --git a/docs/api/puppeteer.createbrowserfetcher.md b/docs/api/puppeteer.createbrowserfetcher.md
@@ -7,6 +7,7 @@ sidebar_label: createBrowserFetcher
 **Signature:**
 
 ```typescript
-createBrowserFetcher: (options: import('./types.js').BrowserFetcherOptions) =>
-  import('./types.js').BrowserFetcher;
+createBrowserFetcher: (
+  options: import('puppeteer-core/internal/node/BrowserFetcher.js').BrowserFetcherOptions
+) => import('puppeteer-core/internal/node/BrowserFetcher.js').BrowserFetcher;
 ```
diff --git a/docs/api/puppeteer.defaultargs.md b/docs/api/puppeteer.defaultargs.md
@@ -7,5 +7,5 @@ sidebar_label: defaultArgs
 **Signature:**
 
 ```typescript
-defaultArgs: (options?: import("./types.js").BrowserLaunchArgumentOptions) => string[]
+defaultArgs: (options?: import("puppeteer-core/internal/node/LaunchOptions.js").BrowserLaunchArgumentOptions | undefined) => string[]
 ```
diff --git a/docs/api/puppeteer.launch.md b/docs/api/puppeteer.launch.md
@@ -7,6 +7,9 @@ sidebar_label: launch
 **Signature:**
 
 ```typescript
-launch: (options?: import('./types.js').PuppeteerLaunchOptions) =>
-  Promise<import('./types.js').Browser>;
+launch: (
+  options?:
+    | import('puppeteer-core/internal/node/PuppeteerNode.js').PuppeteerLaunchOptions
+    | undefined
+) => Promise<import('./types').Browser>;
 ```
diff --git a/docs/api/puppeteer.mouse.draganddrop.md b/docs/api/puppeteer.mouse.draganddrop.md
@@ -24,8 +24,8 @@ class Mouse {
 
 | Parameter | Type                          | Description                                                                                                                                                                        |
 | --------- | ----------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| start     | [Point](./puppeteer.point.md) |                                                                                                                                                                                    |
-| target    | [Point](./puppeteer.point.md) | point to drag from                                                                                                                                                                 |
+| start     | [Point](./puppeteer.point.md) | point to drag from                                                                                                                                                                 |
+| target    | [Point](./puppeteer.point.md) | point to drop on                                                                                                                                                                   |
 | options   | { delay?: number; }           | <i>(Optional)</i> An object of options. Accepts delay which, if specified, is the time to wait between <code>dragover</code> and <code>drop</code> in milliseconds. Defaults to 0. |
 
 **Returns:**

diff --git a/docs/contributing.md b/docs/contributing.md
@@ -42,15 +42,15 @@ Puppeteer has two configurations for building:
 
 - `npm run build` (or `npm run build:prod`) - Builds Puppeteer and artifacts
   used in production.
-- `npm run build:dev` - Builds Puppeteer, test runner, tests, and artifacts used in
+- `npm run build:test` - Builds Puppeteer, test runner, tests, and artifacts used in
   production.
 
 ## Testing Puppeteer
 
 For browser testing, you can run
 
 ```bash
-npm run build:dev && npm run test
+npm run build:test && npm run test
 ```
 
 We also have other tests such as `test:types` that tests types and
@@ -274,7 +274,7 @@ The following steps are needed to update the Chromium version.
 1. Update `versions.js` with the new Chromium-to-Puppeteer version mapping and update `lastMaintainedChromiumVersion` with the latest stable Chrome version.
 1. Run `npm run check:protocol-revision`.
    If it fails, update `package.json` with the expected `devtools-protocol` version.
-1. Run `npm run build && npm run build:dev` and `npm install`.
+1. Run `npm run build && npm run build:test` and `npm install`.
 1. Run `npm test` and ensure that all tests pass. If a test fails, [bisect](#bisecting-upstream-changes) the upstream cause of the failure, and either update the test expectations accordingly (if it was an intended change) or work around the changes in Puppeteer (if it’s not desirable to change Puppeteer’s observable behavior).
 1. Commit and push your changes and open a pull request.
    The commit message must contain the version in `Chromium <version> (<revision>)` format to ensure that [pptr.dev](https://pptr.dev/) can parse it correctly, e.g. `'feat(chromium): roll to Chromium 90.0.4427.0 (r856583)'`.

diff --git a/lerna.json b/lerna.json