refactor: migrate to Yarn Berry (v3.2.1) (#8764)

* Yarn berry

* Add `.yarn` to dockerignore

* Mention in CHANGELOG

* Update Dockerfile

* Proper upgrade to yarn

* use `"packageManager"`

* Replace `install-packages.js` with workspaces

* Replace `install-packages.js` with workspaces

* Typo

* `corepack enable` for linting

* Remove `packages/*/yarn.lock`

* Improve lint workflow

* Update .github/workflows/lint.yml

Co-authored-by: Acid Chicken (硫酸鶏) <root@acid-chicken.com>

* Update .github/workflows/lint.yml

Co-authored-by: Acid Chicken (硫酸鶏) <root@acid-chicken.com>

* Add `eslint`

* Regenerate yarn.lock

* 🙏‍

* Update lint.yml

:thinking:

* Fix workflow!

* yarn lint my beloved

* corepack enable for mocha/e2e

* Update CHANGELOG.md

* fix the code that depends on node_modules

* tbh

* tbh

* More yarn workspaces stuff

* 🍀

* Change `browser-image-resizer`

* :finnadie:

* :goberserk:

* :godmode:

* Remove yarnrc for workspaces

* 🙏

* Add proper packages for eslint

* Skip extra install

* New steps for test.yml too

* uhh

* Update .dockerignore

* Update lint.yml

* Update test.yml

* Make client/backend lints seperate workflows

* package.json names

* pnpm = dont change paths

* Fix changelog

* :bowtie:

* I don't know why github workflows hates me...

* test

* ?????

* continue-on-error: true

* Fix yarn.lock

Co-authored-by: ishowta <ishowta@gmail.com>

* add @rollup/pluginutils

* add packageExtensions for chartjs_date-fns

Co-authored-by: acid-chicken <root@acid-chicken.com>

* npm run => yarn

* wip

* yarn node => node

依存関係の記載不足の解消がかなり多いため、yarn nodeは使わない

* fix

* fix http-signature

* fix

* fix

* add packageExtensions

* ✌️

* remove `yarn set version berry`

* yarn install --immutable

Co-authored-by: ishowta <ishowta@gmail.com>

* #8764 (comment)

* enable actions/setup-node's global cache

* Update .gitignore

Co-authored-by: iwata <ishowta@gmail.com>

* revival gulp build
https://github.com/misskey-dev/misskey/pull/8764/files/66ab7591bff9b35255219057e33399a06260aa31#r885899944

* fix lockfile

* Update packages/backend/package.json

Co-authored-by: iwata <ishowta@gmail.com>

* remove packagemanager

* `yarn run gulp` in `build` instead of just `gulp`

* Update CHANGELOG.md

* update lockfile

* Add .yarn/cache to gitignore for packages

* ?

* move resolutions

* ✌️

* update lockfie

* update-lockfile

* update yarn.lock

* fix build

* fix

* fix

* fix: add @tensorflow/tfjs

* ダメ

* modify lockfile

* use yarnrc

* update yarnlock

* add jest-mock

* update-lockfile

* update lockfile

* update lockfile

* move jest-mock, update lockfile

Co-authored-by: Acid Chicken (硫酸鶏) <root@acid-chicken.com>
Co-authored-by: tamaina <tamaina@hotmail.co.jp>
Co-authored-by: ishowta <ishowta@gmail.com>

.dockerignore

@@ -13,3 +13,10 @@ node_modules/
 redis/
 files/
 misskey-assets/
+.pnp.*
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions

.github/workflows/lint.yml

@@ -8,32 +8,22 @@ on:
   pull_request:
 
 jobs:
-  backend:
+  lint:
+    strategy:
+      matrix:
+        workspace:
+        - backend
+        - client
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
       with:
+        fetch-depth: 0
         submodules: true
-    - uses: actions/setup-node@v3
+    - uses: actions/setup-node@v3.2.0
       with:
         node-version: 18.x
         cache: 'yarn'
-        cache-dependency-path: |
-          packages/backend/yarn.lock
-    - run: yarn install
-    - run: yarn --cwd ./packages/backend lint
-
-  client:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v2
-      with:
-        submodules: true
-    - uses: actions/setup-node@v3
-      with:
-        node-version: 18.x
-        cache: 'yarn'
-        cache-dependency-path: |
-          packages/client/yarn.lock
-    - run: yarn install
-    - run: yarn --cwd ./packages/client lint
+    - run: corepack enable
+    - run: yarn install --immutable
+    - run: yarn workspace ${{ matrix.workspace }} run lint

.github/workflows/test.yml

@@ -23,6 +23,7 @@ jobs:
         env:
           POSTGRES_DB: test-misskey
           POSTGRES_HOST_AUTH_METHOD: trust
+          YARN_CHECKSUM_BEHAVIOR: update
       redis:
         image: redis:6
         ports:
@@ -33,15 +34,12 @@ jobs:
       with:
         submodules: true
     - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v3.2.0
       with:
         node-version: ${{ matrix.node-version }}
         cache: 'yarn'
-        cache-dependency-path: |
-          packages/backend/yarn.lock
-          packages/client/yarn.lock
-    - name: Install dependencies
-      run: yarn install
+    - run: corepack enable
+    - run: yarn install --immutable
     - name: Check yarn.lock
       run: git diff --exit-code yarn.lock
     - name: Copy Configure
@@ -89,17 +87,14 @@ jobs:
     #- uses: browser-actions/setup-firefox@latest
     #  if: ${{ matrix.browser == 'firefox' }}
     - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v3.2.0
       with:
         node-version: ${{ matrix.node-version }}
         cache: 'yarn'
-        cache-dependency-path: |
-          packages/backend/yarn.lock
-          packages/client/yarn.lock
-    - name: Install dependencies
-      run: yarn install
-    - name: Check yarn.lock
-      run: git diff --exit-code yarn.lock
+    - run: corepack enable
+    - run: yarn install --immutable
+      env:
+        YARN_CHECKSUM_BEHAVIOR: update
     - name: Copy Configure
       run: cp .github/misskey/test.yml .config
     - name: Build
@@ -111,7 +106,7 @@ jobs:
       uses: cypress-io/github-action@v4
       with:
         install: false
-        start: npm run start:test
+        start: yarn start:test
         wait-on: 'http://localhost:61812'
         headless: false
         browser: ${{ matrix.browser }}

.gitignore

@@ -9,6 +9,17 @@
 node_modules
 report.*.json
 
+# Yarn
+.yarn/*
+!.yarn/patches
+!.yarn/plugins
+!.yarn/releases
+!.yarn/sdks
+!.yarn/versions
+packages/client/.yarn/cache
+packages/backend/.yarn/cache
+packages/sw/.yarn/cache
+
 # Cypress
 cypress/screenshots
 cypress/videos

.yarnrc.yml

@@ -0,0 +1,30 @@
+httpTimeout: 600000
+
+nmHoistingLimits: none
+
+nodeLinker: pnpm
+
+plugins:
+  - path: .yarn/plugins/@yarnpkg/plugin-interactive-tools.cjs
+    spec: "@yarnpkg/plugin-interactive-tools"
+  - path: .yarn/plugins/@yarnpkg/plugin-workspace-tools.cjs
+    spec: "@yarnpkg/plugin-workspace-tools"
+
+progressBarStyle: "patrick"
+
+packageExtensions:
+  "chartjs-adapter-date-fns@*":
+    peerDependencies:
+      "date-fns": "*"
+  "@bull-board/api@*":
+    peerDependencies:
+      "@bull-board/ui": "*"
+  "koa-views@*":
+    dependencies:
+      "pug": "*"
+  "consolidate@*":
+    dependencies:
+      "ejs": "*"
+  "@tensorflow/tfjs@*":
+    dependencies:
+      "long": "*"

CHANGELOG.md

@@ -170,6 +170,8 @@ same as 12.112.0
   IPv4/IPv6 is used by default. You can tune this behavior via `redis.family`.
 - Server: Add possibility to log IP addresses of users @syuilo
 - Add additional drive capacity change support @CyberRex0
+- Migrate to Yarn Berry (v3.2.1) @ThatOneCalculator
+	- You may have to `yarn run clean-all` and `yarn set version berry` before running `yarn install` if you're still on yarn classic
 
 ### Bugfixes
 - Server: Fix GenerateVideoThumbnail failed @mei23
@@ -379,7 +381,7 @@ same as 12.112.0
 ## 12.104.0 (2022/02/09)
 
 ### Note
-ビルドする前に`npm run clean`を実行してください。
+ビルドする前に`yarn clean`を実行してください。
 
 このリリースはマイグレーションの規模が大きいため、インスタンスによってはマイグレーションに時間がかかる可能性があります。
 マイグレーションが終わらない場合は、チャートの情報はリセットされてしまいますが`__chart__`で始まるテーブルの**レコード**を全て削除(テーブル自体は消さないでください)してから再度試す方法もあります。

CONTRIBUTING.md

@@ -44,7 +44,7 @@ Thank you for your PR! Before creating a PR, please check the following:
 - Check if there are any documents that need to be created or updated due to this change.
 - If you have added a feature or fixed a bug, please add a test case if possible.
 - Please make sure that tests and Lint are passed in advance.
-  - You can run it with `npm run test` and `npm run lint`. [See more info](#testing)
+  - You can run it with `yarn test` and `yarn lint`. [See more info](#testing)
 - If this PR includes UI changes, please attach a screenshot in the text.
 
 Thanks for your cooperation 🤗
@@ -99,7 +99,7 @@ If your language is not listed in Crowdin, please open an issue.
 ![Crowdin](https://d322cqt584bo4o.cloudfront.net/misskey/localized.svg)
 
 ## Development
-During development, it is useful to use the `npm run dev` command.
+During development, it is useful to use the `yarn dev` command.
 This command monitors the server-side and client-side source files and automatically builds them if they are modified.
 In addition, it will also automatically start the Misskey server process.
 
@@ -119,12 +119,12 @@ Alternatively, prepare an empty (data can be erased) DB and edit `.config/test.y
 
 Run all test.
 ```
-npm run test
+yarn test
 ```
 
 #### Run specify test
 ```
-npm run jest -- foo.ts
+yarn jest -- foo.ts
 ```
 
 ### e2e tests
@@ -257,7 +257,7 @@ MongoDBは`null`で返してきてたので、その感覚で`if (x === null)`
 ### Migration作成方法
 packages/backendで:
 ```sh
-npx typeorm migration:generate -d ormconfig.js -o <migration name>
+yarn dlx typeorm migration:generate -d ormconfig.js -o <migration name>
 ```
 
 - 生成後、ファイルをmigration下に移してください

package.json

@@ -6,32 +6,42 @@
 		"type": "git",
 		"url": "https://github.com/misskey-dev/misskey.git"
 	},
+	"packageManager": "yarn@3.2.1",
+	"workspaces": [
+		"packages/client",
+		"packages/backend",
+		"packages/sw"
+	],
 	"private": true,
 	"scripts": {
-		"postinstall": "node ./scripts/install-packages.js",
-		"build": "node ./scripts/build.js",
+		"build": "yarn workspaces foreach run build && yarn run gulp",
 		"start": "cd packages/backend && node ./built/boot/index.js",
 		"start:test": "cd packages/backend && cross-env NODE_ENV=test node ./built/boot/index.js",
-		"init": "npm run migrate",
+		"init": "yarn migrate",
 		"migrate": "cd packages/backend && npx typeorm migration:run -d ormconfig.js",
-		"migrateandstart": "npm run migrate && npm run start",
+		"migrateandstart": "yarn migrate && yarn start",
 		"gulp": "gulp build",
-		"watch": "npm run dev",
+		"watch": "yarn dev",
 		"dev": "node ./scripts/dev.js",
-		"lint": "node ./scripts/lint.js",
+		"lint": "yarn workspaces foreach run lint",
 		"cy:open": "cypress open --browser --e2e --config-file=cypress.config.ts",
 		"cy:run": "cypress run",
 		"e2e": "start-server-and-test start:test http://localhost:61812 cy:run",
 		"jest": "cd packages/backend && cross-env NODE_ENV=test node --experimental-vm-modules --experimental-import-meta-resolve node_modules/jest/bin/jest.js --forceExit --runInBand",
 		"jest-and-coverage": "cd packages/backend && cross-env NODE_ENV=test node --experimental-vm-modules --experimental-import-meta-resolve node_modules/jest/bin/jest.js --coverage --forceExit --runInBand",
-		"test": "npm run jest",
-		"test-and-coverage": "npm run jest-and-coverage",
+		"test": "yarn jest",
+		"test-and-coverage": "yarn jest-and-coverage",
 		"format": "gulp format",
 		"clean": "node ./scripts/clean.js",
 		"clean-all": "node ./scripts/clean-all.js",
-		"cleanall": "npm run clean-all"
+		"cleanall": "yarn clean-all"
+	},
+	"resolutions": {
+		"chokidar": "^3.3.1",
+		"lodash": "^4.17.21"
 	},
 	"dependencies": {
+		"eslint": "^8.16.0",
 		"execa": "5.1.1",
 		"gulp": "4.0.2",
 		"gulp-cssnano": "2.1.3",
@@ -43,6 +53,7 @@
 	"devDependencies": {
 		"@types/gulp": "4.0.10",
 		"@types/gulp-rename": "2.0.1",
+		"@typescript-eslint/eslint-plugin": "latest",
 		"@typescript-eslint/parser": "5.43.0",
 		"cross-env": "7.0.3",
 		"cypress": "11.1.0",

packages/backend/package.json

@@ -1,28 +1,32 @@
 {
+	"name": "backend",
+	"main": "./index.js",
 	"private": true,
 	"type": "module",
 	"scripts": {
+		"start": "node --experimental-json-modules ./built/index.js",
+		"start:test": "NODE_ENV=test node --experimental-json-modules ./built/index.js",
+		"migrate": "typeorm migration:run -d ormconfig.js",
 		"build": "tsc -p tsconfig.json || echo done. && tsc-alias -p tsconfig.json",
 		"watch": "node watch.mjs",
 		"lint": "eslint --quiet \"src/**/*.ts\"",
 		"jest": "cross-env NODE_ENV=test node --experimental-vm-modules --experimental-import-meta-resolve node_modules/jest/bin/jest.js --forceExit --runInBand",
 		"jest-and-coverage": "cross-env NODE_ENV=test node --experimental-vm-modules --experimental-import-meta-resolve node_modules/jest/bin/jest.js --coverage --forceExit --runInBand",
 		"jest-clear": "cross-env NODE_ENV=test node --experimental-vm-modules --experimental-import-meta-resolve node_modules/jest/bin/jest.js --clearCache",
-		"test": "npm run jest",
-		"test-and-coverage": "npm run jest-and-coverage"
-	},
-	"resolutions": {
-		"chokidar": "^3.3.1",
-		"lodash": "^4.17.21"
+		"test": "yarn jest",
+		"test-and-coverage": "yarn jest-and-coverage"
 	},
 	"optionalDependencies": {
+		"@tensorflow/tfjs": "^4.0.0",
 		"@tensorflow/tfjs-node": "4.0.0"
 	},
 	"dependencies": {
+		"@bull-board/api": "4.3.1",
 		"@bull-board/koa": "4.3.1",
+		"@bull-board/ui": "4.3.1",
 		"@discordapp/twemoji": "14.0.2",
-		"@elastic/elasticsearch": "7.11.0",
-		"@koa/cors": "3.1.0",
+		"@elastic/elasticsearch": "7.17.0",
+		"@koa/cors": "3.3.0",
 		"@koa/multer": "3.0.0",
 		"@koa/router": "9.0.1",
 		"@nestjs/common": "9.2.0",
@@ -53,6 +57,7 @@
 		"feed": "4.2.2",
 		"file-type": "18.0.0",
 		"fluent-ffmpeg": "2.1.2",
+		"form-data": "^4.0.0",
 		"got": "12.5.3",
 		"hpagent": "1.2.0",
 		"ioredis": "4.28.5",
@@ -82,6 +87,7 @@
 		"node-fetch": "3.3.0",
 		"nodemailer": "6.8.0",
 		"nsfwjs": "2.4.2",
+		"oauth": "^0.10.0",
 		"os-utils": "0.0.14",
 		"parse5": "7.1.1",
 		"pg": "8.8.0",
@@ -103,6 +109,7 @@
 		"rxjs": "7.5.7",
 		"s-age": "1.1.2",
 		"sanitize-html": "2.7.3",
+		"seedrandom": "^3.0.5",
 		"semver": "7.3.8",
 		"sharp": "0.29.3",
 		"speakeasy": "2.0.0",
@@ -150,7 +157,7 @@
 		"@types/koa-mount": "4.0.1",
 		"@types/koa-send": "4.1.3",
 		"@types/koa-views": "7.0.0",
-		"@types/koa__cors": "3.1.1",
+		"@types/koa__cors": "3.3.0",
 		"@types/koa__multer": "2.0.4",
 		"@types/koa__router": "8.0.11",
 		"@types/mime-types": "2.1.1",
@@ -185,6 +192,7 @@
 		"eslint-plugin-import": "2.26.0",
 		"execa": "6.1.0",
 		"jest": "29.3.1",
+		"jest-mock": "^29.0.3",
 		"typescript": "4.9.3"
 	}
 }

packages/backend/test/tests/ap-request.ts

@@ -1,5 +1,5 @@
 import * as assert from 'assert';
-import httpSignature from 'http-signature';
+import httpSignature from '@peertube/http-signature';
 import { genRsaKeyPair } from '../../src/misc/gen-key-pair.js';
 import { createSignedPost, createSignedGet } from '../../src/remote/activitypub/ap-request.js';