Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

build(docker): optimize; add Chromium bundled ver #9626

Merged
merged 8 commits into from
Apr 28, 2022
Merged

build(docker): optimize; add Chromium bundled ver #9626

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
8a9b6e9
build(docker): optimize; add Chromium bundled ver
Rongronggg9 Apr 23, 2022
af8e685
build(Dockerfile): fix redundant command and label
Rongronggg9 Apr 26, 2022
a1bfdbd
build(docker): only install prod deps to speed up
Rongronggg9 Apr 26, 2022
3494129
docs(install): minor fixes
Rongronggg9 Apr 26, 2022
13efc4e
build(Dockerfile): bump to `node:16-bullseye-slim`
Rongronggg9 Apr 26, 2022
a511f73
build(Dockerfile): fix yarn registry mirror
Rongronggg9 Apr 26, 2022
0d9dc04
build(Dockerfile): minimize dependencies
Rongronggg9 Apr 26, 2022
b6be004
Merge branch 'upstream-master'
Rongronggg9 Apr 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .dockerignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,18 @@ test
process.json
app.json
.travis.yml
app-minimal
.idea
.env
.editorconfig
Procfile
now.json
jsconfig.json
package-lock.json

#git but keep the git commit hash
.git/logs
.git/objects
.git/index
.git/info
.git/hooks
54 changes: 41 additions & 13 deletions .github/workflows/docker-release.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ on:
- 'lib/**'
- '!**/maintainer.js'
- '!**/radar.js'
- '!**/radar-rules.js.js'
- '!**/radar-rules.js'
- 'Dockerfile'
- 'package.json'
- 'yarn.lock'
Expand Down Expand Up @@ -46,8 +46,8 @@ jobs:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}

- name: Extract Docker metadata
id: meta
- name: Extract Docker metadata (ordinary version)
id: meta-ordinary
uses: docker/metadata-action@v3
with:
images: ${{ secrets.DOCKER_USERNAME }}/rsshub
Expand All @@ -56,26 +56,54 @@ jobs:
type=raw,value={{date 'YYYY-MM-DD'}},enable=true
flavor: latest=false

- name: Build and push Docker image
- name: Build and push Docker image (ordinary version)
uses: docker/build-push-action@v2
with:
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
tags: ${{ steps.meta-ordinary.outputs.tags }}
labels: ${{ steps.meta-ordinary.outputs.labels }}
platforms: linux/amd64,linux/arm/v7,linux/arm64
cache-from: type=gha,scope=docker-release
cache-to: type=gha,mode=max,scope=docker-release
# cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/rsshub:buildcache
# cache-to: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/rsshub:buildcache,mode=max

- name: Extract Docker metadata (Chromium-bundled version)
id: meta-chromium-bundled
uses: docker/metadata-action@v3
with:
images: ${{ secrets.DOCKER_USERNAME }}/rsshub
tags: |
type=raw,value=chromium-bundled,enable=true
type=raw,value=chromium-bundled-{{date 'YYYY-MM-DD'}},enable=true
flavor: latest=false

- name: Build and push Docker image (Chromium-bundled version)
uses: docker/build-push-action@v2
with:
context: .
build-args: PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=0
push: true
tags: ${{ steps.meta-chromium-bundled.outputs.tags }}
labels: ${{ steps.meta-chromium-bundled.outputs.labels }}
platforms: linux/amd64 # bundled Chromium is only available on amd64
cache-from: |
type=gha,scope=docker-release
type=registry,ref=${{ secrets.DOCKER_USERNAME }}/rsshub:chromium-bundled
# type=registry,ref=${{ secrets.DOCKER_USERNAME }}/rsshub:buildcache
cache-to: type=inline,ref=${{ secrets.DOCKER_USERNAME }}/rsshub:chromium-bundled # inline cache is enough

description:
runs-on: ubuntu-latest
needs: check-env
if: needs.check-env.outputs.check-docker == 'true'
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v3

- name: Docker Hub Description
uses: peter-evans/dockerhub-description@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
repository: ${{ secrets.DOCKER_USERNAME }}/rsshub
- name: Docker Hub Description
uses: peter-evans/dockerhub-description@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
repository: ${{ secrets.DOCKER_USERNAME }}/rsshub
10 changes: 8 additions & 2 deletions .github/workflows/docker-test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,19 @@ jobs:
uses: docker/build-push-action@v2
with:
context: .
build-args: PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=0 # also test bundling Chromium
load: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
platforms: linux/amd64 # explicit
cache-from: |
type=gha,scope=docker-test
type=registry,ref=${{ secrets.DOCKER_USERNAME }}/rsshub:chromium-bundled
type=gha,scope=docker-release
cache-to: type=gha,mode=max,scope=docker-test
# ! build on amd64 is fast enough, and cache between PRs never hit, so never waste the 10GB cache limit !
# cache-from: |
# type=gha,scope=docker-test
# type=gha,scope=docker-release
# cache-to: type=gha,mode=max,scope=docker-test

- name: Run dockerfile
run: |
Expand Down
208 changes: 164 additions & 44 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,65 +1,185 @@
FROM node:14-buster-slim as dep-builder
FROM node:16-bullseye-slim as dep-builder

LABEL MAINTAINER https://github.com/DIYgod/RSSHub/
# bash has already been the default shell
#RUN ln -sf /bin/bash /bin/sh

ARG USE_CHINA_NPM_REGISTRY=0
ARG PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1

RUN ln -sf /bin/bash /bin/sh
RUN apt-get update && apt-get install -yq libgconf-2-4 apt-transport-https git dumb-init python3 build-essential --no-install-recommends
# these deps are no longer needed since we use yarn instead of npm to install dependencies
# the installation of dumb-init has been moved to the app stage to improve concurrency and speed up builds on arm/arm64
#RUN \
# set -ex && \
# apt-get update && \
# apt-get install -yq --no-install-recommends \
# libgconf-2-4 apt-transport-https git dumb-init python3 build-essential \
# && \
# rm -rf /var/lib/apt/lists/*

WORKDIR /app

COPY ./yarn.lock /app
COPY ./package.json /app
# place ARG statement before RUN statement which need it to avoid cache miss
ARG USE_CHINA_NPM_REGISTRY=0
RUN \
set -ex && \
if [ "$USE_CHINA_NPM_REGISTRY" = 1 ]; then \
echo 'use npm mirror' && \
npm config set registry https://registry.npmmirror.com && \
yarn config set registry https://registry.npmmirror.com ; \
fi;

COPY ./yarn.lock /app/
COPY ./package.json /app/

# lazy install Chromium to avoid cache miss, only install production dependencies to minimize the image size
RUN \
set -ex && \
export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true && \
yarn install --production --frozen-lockfile --network-timeout 1000000 && \
yarn cache clean

# ---------------------------------------------------------------------------------------------------------------------

FROM debian:bullseye-slim as dep-version-parser
# This stage is necessary to limit the cache miss scope.
# With this stage, any modification to package.json won't break the build cache of the next two stages as long as the
# version unchanged.
# node:16-bullseye-slim is based on debian:bullseye-slim so this stage would not cause any additional download.

WORKDIR /ver
COPY ./package.json /app/
RUN \
set -ex && \
grep -Po '(?<="puppeteer": ")[^\s"]*(?=")' /app/package.json | tee /ver/.puppeteer_version && \
grep -Po '(?<="@vercel/nft": ")[^\s"]*(?=")' /app/package.json | tee /ver/.nft_version && \
grep -Po '(?<="fs-extra": ")[^\s"]*(?=")' /app/package.json | tee /ver/.fs_extra_version

# ---------------------------------------------------------------------------------------------------------------------

FROM node:16-bullseye-slim as docker-minifier
# The stage is used to further reduce the image size by removing unused files.

WORKDIR /minifier
COPY --from=dep-version-parser /ver/* /minifier/

ARG USE_CHINA_NPM_REGISTRY=0
RUN \
set -ex && \
if [ "$USE_CHINA_NPM_REGISTRY" = 1 ]; then \
npm config set registry https://registry.npmmirror.com && \
yarn config set registry https://registry.npmmirror.com ; \
fi; \
yarn add @vercel/nft@$(cat .nft_version) fs-extra@$(cat .fs_extra_version) && \
yarn cache clean

RUN if [ "$USE_CHINA_NPM_REGISTRY" = 1 ]; then \
echo 'use npm mirror'; npm config set registry https://registry.npmmirror.com; \
fi;
COPY . /app
COPY --from=dep-builder /app /app

RUN npm i -g npm
RUN \
set -ex && \
cp /app/scripts/docker/minify-docker.js /minifier/ && \
export PROJECT_ROOT=/app && \
node /minifier/minify-docker.js && \
rm -rf /app/node_modules /app/scripts && \
mv /app/app-minimal/node_modules /app/ && \
rm -rf /app/app-minimal && \
ls -la /app && \
du -hd1 /app

RUN if [ "$PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" = 0 ]; then \
unset PUPPETEER_SKIP_CHROMIUM_DOWNLOAD ;\
else \
export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true ;\
fi;
# ---------------------------------------------------------------------------------------------------------------------

RUN yarn --frozen-lockfile --network-timeout 1000000
COPY . /app
RUN node scripts/docker/minify-docker.js
FROM node:16-bullseye-slim as chromium-downloader
# This stage is necessary to improve build concurrency and minimize the image size.
# Yeah, downloading Chromium never needs those dependencies below.

WORKDIR /app
COPY --from=dep-version-parser /ver/.puppeteer_version /app/.puppeteer_version

FROM node:14-slim as app
ARG USE_CHINA_NPM_REGISTRY=0
ARG PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1
# https://github.com/puppeteer/puppeteer#q-why-doesnt-puppeteer-vxxx-work-with-chromium-vyyy
RUN \
set -ex ; \
if [ "$PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" = 0 ]; then \
if [ "$USE_CHINA_NPM_REGISTRY" = 1 ]; then \
npm config set registry https://registry.npmmirror.com && \
yarn config set registry https://registry.npmmirror.com ; \
fi; \
echo 'Downloading Chromium...' && \
unset PUPPETEER_SKIP_CHROMIUM_DOWNLOAD && \
yarn add puppeteer@$(cat /app/.puppeteer_version) && \
yarn cache clean ; \
else \
mkdir -p /app/node_modules/puppeteer ; \
fi;

# ---------------------------------------------------------------------------------------------------------------------

FROM node:16-bullseye-slim as app

LABEL org.opencontainers.image.authors="https://github.com/DIYgod/RSSHub"

ENV NODE_ENV production
ENV TZ Asia/Shanghai
ARG PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1

WORKDIR /app
COPY . /app
COPY --from=dep-builder /app/app-minimal/node_modules /app/node_modules
COPY --from=dep-builder /usr/bin/dumb-init /usr/bin/dumb-init

RUN if [ "$PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" = 0 ]; then \
apt-get update \
&& apt-get install -y wget gnupg ca-certificates --no-install-recommends \
&& wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
&& sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \
&& set -ex \
&& apt-get update \
&& apt-get install -y google-chrome-unstable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf \
ca-certificates \
fonts-liberation libappindicator3-1 libasound2 libatk-bridge2.0-0 libatk1.0-0 libc6 libcairo2 libcups2 libdbus-1-3 libexpat1 \
libfontconfig1 libgbm1 libgcc1 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 libpango-1.0-0 libpangocairo-1.0-0 libstdc++6 \
libx11-6 libx11-xcb1 libxcb1 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxi6 libxrandr2 \
libxrender1 libxss1 libxtst6 lsb-release \
--no-install-recommends \
&& rm -rf /var/lib/apt/lists/* \
&& apt-get purge --auto-remove -y wget gnupg; \
fi;

# install deps first to avoid cache miss or disturbing buildkit to build concurrently
ARG PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=1
# https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md#chrome-headless-doesnt-launch-on-unix
# https://github.com/puppeteer/puppeteer/issues/7822
# https://www.debian.org/releases/bullseye/amd64/release-notes/ch-information.en.html#noteworthy-obsolete-packages
RUN \
set -ex && \
apt-get update && \
apt-get install -yq --no-install-recommends \
dumb-init \
; \
if [ "$PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" = 0 ]; then \
apt-get install -yq --no-install-recommends \
ca-certificates fonts-liberation wget xdg-utils \
libasound2 libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libcairo2 libcups2 libdbus-1-3 libdrm2 libexpat1 \
libgbm1 libglib2.0-0 libnspr4 libnss3 libpango-1.0-0 libx11-6 libxcb1 libxcomposite1 libxdamage1 libxext6 \
libxfixes3 libxkbcommon0 libxrandr2 \
; \
fi; \
rm -rf /var/lib/apt/lists/*

COPY --from=chromium-downloader /app/node_modules/puppeteer /app/node_modules/puppeteer

# if grep matches nothing then it will exit with 1, thus, we cannot `set -e` here
RUN \
set -x && \
if [ "$PUPPETEER_SKIP_CHROMIUM_DOWNLOAD" = 0 ]; then \
echo 'Verifying Chromium installation...' && \
ldd $(find /app/node_modules/puppeteer/ -name chrome) | grep "not found" ; \
if [ "$?" = 0 ]; then \
echo "!!! Chromium has unmet shared libs !!!" && \
exit 1 ; \
else \
echo "Awesome! All shared libs are met!" ; \
fi; \
fi;

COPY --from=docker-minifier /app /app

EXPOSE 1200
ENTRYPOINT ["dumb-init", "--"]

CMD ["npm", "run", "start"]

# ---------------------------------------------------------------------------------------------------------------------

# In case Chromium has unmet shared libs, here is some magic to find and install the packages they belong to:
# In most case you can just stop at `grep ^lib` and add those packages to the above stage.
#
# apt-get update && \
# apt install -yq --no-install-recommends \
# apt-file \
# && \
# apt-file update && \
# ldd $(find /app/node_modules/puppeteer/ -name chrome) | grep -Po "\S+(?= => not found)" | \
# sed 's/\./\\./g' | awk '{print $1"$"}' | apt-file search -xlf - | grep ^lib | \
# xargs -d '\n' -- \
# apt-get install -yq --no-install-recommends \
# && \
# apt purge -yq --auto-remove \
# apt-file \
# rm -rf /tmp/.chromium_path /var/lib/apt/lists/*
21 changes: 12 additions & 9 deletions docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@ version: '3'

services:
rsshub:
# two ways to enable puppeteer:
# * (only on amd64/x86_64) comment out marked lines, then use this image instead: diygod/rsshub:chromium-bundled
# * (on all supported architectures, but consumes more disk space and memory) leave anything unchanged
image: diygod/rsshub
restart: always
ports:
Expand All @@ -10,18 +13,18 @@ services:
NODE_ENV: production
CACHE_TYPE: redis
REDIS_URL: 'redis://redis:6379/'
PUPPETEER_WS_ENDPOINT: 'ws://browserless:3000'
PUPPETEER_WS_ENDPOINT: 'ws://browserless:3000' # marked
depends_on:
- redis
- browserless
- browserless # marked

browserless:
image: browserless/chrome
restart: always
ulimits:
core:
hard: 0
soft: 0
browserless: # marked
image: browserless/chrome # marked
restart: always # marked
ulimits: # marked
core: # marked
hard: 0 # marked
soft: 0 # marked

redis:
image: redis:alpine
Expand Down
Loading