diff --git a/.config/example.yml b/.config/example.yml index 6eda053a497e..e4ed342c7d11 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -295,5 +295,10 @@ signToActivityPubGet: true # Upload or download file size limits (bytes) # maxFileSize: 262144000 +# timeout and maximum size for imports (e.g. note imports) +#import: +# downloadTimeout: 30 +# maxFileSize: 262144000 + # PID File of master process # pidFile: /tmp/misskey.pid diff --git a/locales/en-US.yml b/locales/en-US.yml index fe2bb08074fa..ab1fb6082ae6 100644 --- a/locales/en-US.yml +++ b/locales/en-US.yml @@ -73,6 +73,10 @@ mentions: "Mentions" directNotes: "Direct notes" importAndExport: "Import / Export" import: "Import" +importOrigin: "Import Source" +importNoteInfo: "You can import notes exported from other services." +importNoteDisclaimer: "Not all notes can be imported. In cases where Misskey has been modified, or the export is from a completely different platform, import may not be possible at all." +importNoteWarm: "Imported notes may only be visible on your user page." export: "Export" files: "Files" download: "Download" @@ -1533,6 +1537,9 @@ _achievements: title: "I Am a Cat" description: "Mark your account as a cat" flavor: "I'll give you a name later." + _markedAsHanaModeUser: + title: "Wish I were born a small person like a violet" + description: "Unlocked Hana Mode to take the first step of your tiny, comfy fedi experience" _following1: title: "Following your first user" description: "Follow a user" @@ -1707,6 +1714,7 @@ _role: ltlAvailable: "Can view the local timeline" canPublicNote: "Can send public notes" mentionMax: "Maximum number of mentions in a note" + canImportNotes: "Can import notes" canInvite: "Can create instance invite codes" inviteLimit: "Invite limit" inviteLimitCycle: "Invite limit cooldown" @@ -2263,6 +2271,7 @@ _instanceCharts: filesTotal: "Cumulative number of files" _timelines: home: "Home" + hanami: "Hanami" local: "Local" social: "Social" global: "Global" @@ -2635,3 +2644,42 @@ _contextMenu: app: "Application" appWithShift: "Application with shift key" native: "Native" +_hana: + hanaSettings: "HanaMisskey Settings" + hanaMode: "Hana Mode" + _inDevelopment: + title: "This feature is under development" + description: "HanaMisskey is packed with new features currently in development! Stay tuned to see what exciting features will be implemented." + _welcome: + whatAboutX: "What is '{x}'?" + _aboutHana: + title: "HanaMisskey" + description: "HanaMisskey is a decentralized social networking service (SNS) based on Misskey. It offers a variety of unique features to enhance your social experience." + _aboutDecentralized: + title: "Decentralized SNS" + description: "Traditional SNS services (such as X, Instagram, YouTube) are self-contained within their own networks. However, decentralized SNS services connect with each other through a common protocol, allowing you to view and follow posts from users on other services. HanaMisskey supports the decentralized technology ActivityPub by default, enabling communication with other services using Misskey, Mastodon or Threads." + _features: + inDevelopment: "Coming Soon" + _hanaMode: + title: "Enjoy a personalized SNS experience with Hana Mode" + description: "When you enable HanaMisskey's unique 'Hana Mode,' your posts won't appear in the Local Timeline (LTL). However, unlike 'Home' posts, they can still be re-noted to LTL and will be delivered as standard public posts to external servers. This feature allows you to create a decentralized SNS experience similar to a private server with just one click." + _easyMigration: + title: "Easy Migration from Other Services" + description: "We offer a feature that makes it easy to migrate from other Misskey servers! You can even carry over your past posts to HanaMisskey." + _preciseSearching: + title: "High-Precision Search Powered by the Latest Technology" + description: "You can take advantage of high-precision, fast search capabilities, fine-tuned for HanaMisskey based on machine learning and the latest academic research. As HanaMisskey's search function is itself a part of a research project, its accuracy is expected to continuously improve." + _cta: + title: "Let Your SNS Experience Blossom with HanaMisskey!" + _hanaModeSwitcher: + recommendedFor: "Recommended for" + normal: "Normal" + normal1: "You can use the LTL (Local Timeline)" + normal2: "Posts set to 'Public' will appear on the LTL" + normalRecommend: "For those who want to prioritize interaction with users on the same server" + hana1: "LTL is not available" + hana2: "Posts set to 'Public' will not appear on the LTL" + hana3: "You can use the 'Hana Timeline,' which allows you to browse the latest notes from users you follow and popular posts from the Fediverse" + hanaRecommend: "For those who want a decentralized SNS experience similar to a private server, while still emphasizing interaction with external servers as well" + saveConfirmDescription: "There is a limit on how many times you can switch modes within a certain period." + diff --git a/locales/index.d.ts b/locales/index.d.ts index 6d486b7f991c..2c782e5ab108 100644 --- a/locales/index.d.ts +++ b/locales/index.d.ts @@ -308,6 +308,22 @@ export interface Locale extends ILocale { * インポート */ readonly "import": string; + /** + * インポート元 + */ + "importOrigin": string; + /** + * 他サービスでエクスポートしたノートなどをインポートすることができます。 + */ + "importNoteInfo": string; + /** + * すべてのノートがインポートできるわけではありません。改変のないMisskey以外では全くインポートできない場合もあります。 + */ + "importNoteDisclaimer": string; + /** + * インポートされたノートは、ユーザーページ以外には表示されない場合があります。 + */ + "importNoteWarm": string; /** * エクスポート */ @@ -5500,6 +5516,10 @@ export interface Locale extends ILocale { }; }; readonly "_timelineDescription": { + /** + * はなみタイムラインでは、フォローしているアカウントの投稿に加えて、連合しているサーバーの人気な投稿も見られます。 + */ + "hanami": string; /** * ホームタイムラインでは、あなたがフォローしているアカウントの投稿を見られます。 */ @@ -6047,7 +6067,17 @@ export interface Locale extends ILocale { */ readonly "flavor": string; }; - readonly "_following1": { + "_markedAsHanaModeUser": { + /** + * 菫ほどな小さき人に生まれたし + */ + "title": string; + /** + * はなモードを有効にした + */ + "description": string; + }; + "_following1": { /** * はじめてのフォロー */ @@ -6647,6 +6677,10 @@ export interface Locale extends ILocale { * ローカルタイムラインの閲覧 */ readonly "ltlAvailable": string; + /** + * はなみタイムラインの閲覧 + */ + "hanamiTlAvailable": string; /** * パブリック投稿の許可 */ @@ -6655,6 +6689,10 @@ export interface Locale extends ILocale { * ノート内の最大メンション数 */ readonly "mentionMax": string; + /** + * ノートのインポート + */ + "canImportNotes": string; /** * サーバー招待コードの発行 */ @@ -6769,6 +6807,10 @@ export interface Locale extends ILocale { * botユーザー */ readonly "isBot": string; + /** + * はなモードが有効なユーザー + */ + "isInHanaMode": string; /** * サスペンド済みユーザー */ @@ -8786,6 +8828,10 @@ export interface Locale extends ILocale { * ホーム */ readonly "home": string; + /** + * はなみ + */ + "hanami": string; /** * ローカル */ @@ -10194,6 +10240,7 @@ export interface Locale extends ILocale { */ readonly "native": string; }; +<<<<<<< HEAD readonly "_tms": { /** * taiy @@ -10312,6 +10359,22 @@ export interface Locale extends ILocale { */ readonly "memoIsNotShared": string; readonly "_about": { +======= + "_hana": { + /** + * はなみすきー設定 + */ + "hanaSettings": string; + /** + * はなモード + */ + "hanaMode": string; + /** + * はな + */ + "hanaModeShort": string; + "_inDevelopment": { +>>>>>>> 15448033b2 (Feat:はなモード / ノートインポート (#23)) /** * taiymeについて */ @@ -10489,6 +10552,48 @@ export interface Locale extends ILocale { */ readonly "repositoryUrlDescription": string; }; + "_hanaModeSwitcher": { + /** + * こんな方におすすめ + */ + "recomenddedFor": string; + /** + * 通常 + */ + "normal": string; + /** + * LTLが使えます + */ + "normal1": string; + /** + * 公開範囲「パブリック」で投稿した内容はLTLに表示されます + */ + "normal2": string; + /** + * サーバー内のユーザーとの交流を重視したい方 + */ + "normalRecommend": string; + /** + * LTLが使えません + */ + "hana1": string; + /** + * 公開範囲「パブリック」で投稿した内容はLTLに表示されません + */ + "hana2": string; + /** + * フォロー中ユーザーの最新のノートとFediverseの人気の投稿をザッピングできる「はなみタイムライン」が使用できます + */ + "hana3": string; + /** + * おひとりさまサーバーのような分散SNS体験をしたい方（内々での交流だけでなく、外部サーバーとの交流もしっかり重視したい方） + */ + "hanaRecommend": string; + /** + * 一定期間にモードを変更できる回数には制限があります。 + */ + "saveConfirmDescription": string; + }; }; } declare const locales: { diff --git a/locales/ja-JP.yml b/locales/ja-JP.yml index 1c2546a1759f..0af942c15c67 100644 --- a/locales/ja-JP.yml +++ b/locales/ja-JP.yml @@ -73,6 +73,10 @@ mentions: "あなた宛て" directNotes: "ダイレクト投稿" importAndExport: "インポートとエクスポート" import: "インポート" +importOrigin: "インポート元" +importNoteInfo: "他サービスでエクスポートしたノートなどをインポートすることができます。" +importNoteDisclaimer: "すべてのノートがインポートできるわけではありません。改変のないMisskey以外では全くインポートできない場合もあります。" +importNoteWarm: "インポートされたノートは、ユーザーページ以外には表示されない場合があります。" export: "エクスポート" files: "ファイル" download: "ダウンロード" @@ -1387,6 +1391,7 @@ _initialTutorial: description: "ここで紹介した機能はほんの一部にすぎません。Misskeyの使い方をより詳しく知るには、{link}をご覧ください。" _timelineDescription: + hanami: "はなみタイムラインでは、フォローしているアカウントの投稿に加えて、連合しているサーバーの人気な投稿も見られます。" home: "ホームタイムラインでは、あなたがフォローしているアカウントの投稿を見られます。" local: "ローカルタイムラインでは、このサーバーにいるユーザー全員の投稿を見られます。" social: "ソーシャルタイムラインには、ホームタイムラインとローカルタイムラインの投稿が両方表示されます。" @@ -1545,6 +1550,9 @@ _achievements: title: "吾輩は猫である" description: "アカウントをCatとして設定した" flavor: "名前はまだない。" + _markedAsHanaModeUser: + title: "菫ほどな小さき人に生まれたし" + description: "はなモードを有効にした" _following1: title: "はじめてのフォロー" description: "初めてフォローした" @@ -1718,8 +1726,10 @@ _role: _options: gtlAvailable: "グローバルタイムラインの閲覧" ltlAvailable: "ローカルタイムラインの閲覧" + hanamiTlAvailable: "はなみタイムラインの閲覧" canPublicNote: "パブリック投稿の許可" mentionMax: "ノート内の最大メンション数" + canImportNotes: "ノートのインポート" canInvite: "サーバー招待コードの発行" inviteLimit: "招待コードの作成可能数" inviteLimitCycle: "招待コードの発行間隔" @@ -1749,6 +1759,7 @@ _role: isRemote: "リモートユーザー" isCat: "猫ユーザー" isBot: "botユーザー" + isInHanaMode: "はなモードが有効なユーザー" isSuspended: "サスペンド済みユーザー" isLocked: "鍵アカウントユーザー" isExplorable: "「アカウントを見つけやすくする」が有効なユーザー" @@ -2318,6 +2329,7 @@ _instanceCharts: _timelines: home: "ホーム" + hanami: "はなみ" local: "ローカル" social: "ソーシャル" global: "グローバル" @@ -2798,3 +2810,43 @@ _tms: caption: "長押しを含む操作が中断される問題を解消します。" _admin: repositoryUrlDescription: "ソースコードが公開されているリポジトリがある場合、そのURLを記入します。taiymeを現状のまま（ソースコードにいかなる変更も加えずに）使用している場合は https://github.com/taiyme/misskey と記入します。" +_hana: + hanaSettings: "はなみすきー設定" + hanaMode: "はなモード" + hanaModeShort: "はな" + _inDevelopment: + title: "この機能は開発中です" + description: "はなみすきーは新規機能盛りだくさんで鋭意開発中です！\nどんな機能が実装されるかはお楽しみ。" + _welcome: + whatAboutX: "「{x}」とは？" + _aboutHana: + title: "はなみすきー" + description: "はなみすきーは、Misskeyベースの分散型SNSサービスです。\nあなたのSNS体験に「はな」を添える、数々の独自機能を備えています。" + _aboutDecentralized: + title: "分散型SNS" + description: "一般的なSNSサービス（X, Instagram, YouTubeなど）はそのサービス内で投稿データなどが完結するようになっています。しかし、分散型SNSは、サービス同士が共通の仕組みを通して連携しており、他のサービスのユーザーの投稿を見たり、フォローしたりできるのです。\nはなみすきーは標準で分散型テクノロジーのActivityPubに対応しており、他のMisskey/Mastodon等を利用したサービスやThreadsなどと通信することができます。" + _features: + inDevelopment: "近日提供予定" + _hanaMode: + title: "はなモードで、あなただけのSNS体験を" + description: "はなみすきー独自機能「はなモード」をオンにすると、投稿内容がローカルタイムライン（LTL）に流れないようになります。ただし、「ホーム」投稿とは違い、LTLへのリノートが可能なほか、外部サーバーには通常のパブリックの投稿として配信されます。\nこれにより、おひとりさまサーバーに近い分散SNS体験をワンクリックで構築することができます。" + _easyMigration: + title: "他サービスから簡単移行" + description: "他のMisskeyサーバーなどからの移行が簡単になる機能を提供！過去の投稿もはなみすきー上に引き継ぐことができます。" + _preciseSearching: + title: "最新技術を活用した高精度な検索" + description: "機械学習や最新の学術研究をもとに、はなみすきーのためにチューニングされた高精度で高速な検索機能を利用できます。また、はなみすきーの検索機能そのものが学術研究プロジェクトとなっているため、検索精度は日進月歩となることが期待できます。" + _cta: + title: "「はな」のあるSNS体験を楽しもう" + _hanaModeSwitcher: + recomenddedFor: "こんな方におすすめ" + normal: "通常" + normal1: "LTLが使えます" + normal2: "公開範囲「パブリック」で投稿した内容はLTLに表示されます" + normalRecommend: "サーバー内のユーザーとの交流を重視したい方" + hana1: "LTLが使えません" + hana2: "公開範囲「パブリック」で投稿した内容はLTLに表示されません" + hana3: "フォロー中ユーザーの最新のノートとFediverseの人気の投稿をザッピングできる「はなみタイムライン」が使用できます" + hanaRecommend: "おひとりさまサーバーのような分散SNS体験をしたい方（内々での交流だけでなく、外部サーバーとの交流もしっかり重視したい方）" + saveConfirmDescription: "一定期間にモードを変更できる回数には制限があります。" +>>>>>>> 15448033b2 (Feat:はなモード / ノートインポート (#23)) diff --git a/packages/backend/migration/1723641187454-AddIsInHanaModeColumnToMiUser.js b/packages/backend/migration/1723641187454-AddIsInHanaModeColumnToMiUser.js new file mode 100644 index 000000000000..4c215ef5465b --- /dev/null +++ b/packages/backend/migration/1723641187454-AddIsInHanaModeColumnToMiUser.js @@ -0,0 +1,13 @@ +export class AddIsInHanaModeColumnToMiUser1723641187454 { + name = 'AddIsInHanaModeColumnToMiUser1723641187454' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "user" ADD "isInHanaMode" boolean NOT NULL DEFAULT false`); + await queryRunner.query(`COMMENT ON COLUMN "user"."isInHanaMode" IS 'Whether the User is in Hana Mode.'`); + } + + async down(queryRunner) { + await queryRunner.query(`COMMENT ON COLUMN "user"."isInHanaMode" IS 'Whether the User is in Hana Mode.'`); + await queryRunner.query(`ALTER TABLE "user" DROP COLUMN "isInHanaMode"`); + } +} diff --git a/packages/backend/migration/1723664940877-AddIsNoteInHanaModeColumnToMiNote.js b/packages/backend/migration/1723664940877-AddIsNoteInHanaModeColumnToMiNote.js new file mode 100644 index 000000000000..0885c4d210e7 --- /dev/null +++ b/packages/backend/migration/1723664940877-AddIsNoteInHanaModeColumnToMiNote.js @@ -0,0 +1,11 @@ +export class AddIsNoteInHanaModeColumnToMiNote1723664940877 { + name = 'AddIsNoteInHanaModeColumnToMiNote1723664940877' + + async up(queryRunner) { + await queryRunner.query(`ALTER TABLE "note" ADD "isNoteInHanaMode" boolean NOT NULL DEFAULT false`); + } + + async down(queryRunner) { + await queryRunner.query(`ALTER TABLE "note" DROP COLUMN "isNoteInHanaMode"`); + } +} diff --git a/packages/backend/src/config.ts b/packages/backend/src/config.ts index 3e5a1e81cd70..372633fd788a 100644 --- a/packages/backend/src/config.ts +++ b/packages/backend/src/config.ts @@ -95,6 +95,12 @@ type Source = { perChannelMaxNoteCacheCount?: number; perUserNotificationsMaxCount?: number; deactivateAntennaThreshold?: number; + + import?: { + downloadTimeout: number; + maxFileSize: number; + }; + pidFile: string; }; @@ -174,6 +180,12 @@ export type Config = { perChannelMaxNoteCacheCount: number; perUserNotificationsMaxCount: number; deactivateAntennaThreshold: number; + + import: { + downloadTimeout: number; + maxFileSize: number; + } | undefined; + pidFile: string; }; @@ -275,6 +287,7 @@ export function loadConfig(): Config { perChannelMaxNoteCacheCount: config.perChannelMaxNoteCacheCount ?? 1000, perUserNotificationsMaxCount: config.perUserNotificationsMaxCount ?? 500, deactivateAntennaThreshold: config.deactivateAntennaThreshold ?? (1000 * 60 * 60 * 24 * 7), + import: config.import, pidFile: config.pidFile, }; } diff --git a/packages/backend/src/core/AchievementService.ts b/packages/backend/src/core/AchievementService.ts index 4fc1193f32aa..73ab12682319 100644 --- a/packages/backend/src/core/AchievementService.ts +++ b/packages/backend/src/core/AchievementService.ts @@ -52,6 +52,7 @@ export const ACHIEVEMENT_TYPES = [ 'myNoteFavorited1', 'profileFilled', 'markedAsCat', + 'markedAsHanaModeUser', 'following1', 'following10', 'following50', diff --git a/packages/backend/src/core/DownloadService.ts b/packages/backend/src/core/DownloadService.ts index 21ae798f9fca..83452845d463 100644 --- a/packages/backend/src/core/DownloadService.ts +++ b/packages/backend/src/core/DownloadService.ts @@ -35,14 +35,14 @@ export class DownloadService { } @bindThis - public async downloadUrl(url: string, path: string): Promise<{ + public async downloadUrl(url: string, path: string, options: { timeout?: number, operationTimeout?: number, maxSize?: number} = {} ): Promise<{ filename: string; }> { this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`); - const timeout = 30 * 1000; - const operationTimeout = 60 * 1000; - const maxSize = this.config.maxFileSize ?? 262144000; + const timeout = options.timeout ?? 30 * 1000; + const operationTimeout = options.operationTimeout ?? 60 * 1000; + const maxSize = options.maxSize ?? this.config.maxFileSize ?? 262144000; const urlObj = new URL(url); let filename = urlObj.pathname.split('/').pop() ?? 'untitled'; diff --git a/packages/backend/src/core/FanoutTimelineService.ts b/packages/backend/src/core/FanoutTimelineService.ts index f6dabfadcd6d..59452f7aacb2 100644 --- a/packages/backend/src/core/FanoutTimelineService.ts +++ b/packages/backend/src/core/FanoutTimelineService.ts @@ -14,9 +14,9 @@ export type FanoutTimelineName = | `homeTimeline:${string}` | `homeTimelineWithFiles:${string}` // only notes with files are included // local timeline - | `localTimeline` // replies are not included - | `localTimelineWithFiles` // only non-reply notes with files are included - | `localTimelineWithReplies` // only replies are included + | 'localTimeline' // replies are not included + | 'localTimelineWithFiles' // only non-reply notes with files are included + | 'localTimelineWithReplies' // only replies are included | `localTimelineWithReplyTo:${string}` // Only replies to specific local user are included. Parameter is reply user id. // antenna @@ -38,6 +38,8 @@ export type FanoutTimelineName = // role timelines | `roleTimeline:${string}` // any notes are included +export type FanoutTimelineNamePrefix = 'homeTimeline' | 'localTimeline' | 'antennaTimeline' | 'userTimeline' | 'userListTimeline' | 'channelTimeline' | 'roleTimeline'; + @Injectable() export class FanoutTimelineService { constructor( diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index c6eeeb50c675..4491986a255d 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -54,7 +54,7 @@ import { RoleService } from '@/core/RoleService.js'; import { MetaService } from '@/core/MetaService.js'; import { SearchService } from '@/core/SearchService.js'; import { FeaturedService } from '@/core/FeaturedService.js'; -import { FanoutTimelineService } from '@/core/FanoutTimelineService.js'; +import { FanoutTimelineNamePrefix, FanoutTimelineService } from '@/core/FanoutTimelineService.js'; import { UtilityService } from '@/core/UtilityService.js'; import { UserBlockingService } from '@/core/UserBlockingService.js'; import { isReply } from '@/misc/is-reply.js'; @@ -135,6 +135,7 @@ type Option = { files?: MiDriveFile[] | null; poll?: IPoll | null; localOnly?: boolean | null; + isNoteInHanaMode?: boolean | null; reactionAcceptance?: MiNote['reactionAcceptance']; cw?: string | null; visibility?: string; @@ -228,7 +229,13 @@ export class NoteCreateService implements OnApplicationShutdown { host: MiUser['host']; isBot: MiUser['isBot']; isCat: MiUser['isCat']; + isInHanaMode: MiUser['isInHanaMode']; }, data: Option, silent = false): Promise { + // ノートのisNoteInHanaMode属性は投稿時のユーザーの属性に基本的に依存する + if (data.isNoteInHanaMode == null) { + data.isNoteInHanaMode = user.isInHanaMode; + } + // チャンネル外にリプライしたら対象のスコープに合わせる // (クライアントサイドでやっても良い処理だと思うけどとりあえずサーバーサイドで) if (data.reply && data.channel && data.reply.channelId !== data.channel.id) { @@ -402,8 +409,190 @@ export class NoteCreateService implements OnApplicationShutdown { return note; } + @bindThis + public async import(user: { + id: MiUser['id']; + username: MiUser['username']; + host: MiUser['host']; + isBot: MiUser['isBot']; + }, data: Option, silent = false): Promise { + // チャンネル外にリプライしたら対象のスコープに合わせる + // (クライアントサイドでやっても良い処理だと思うけどとりあえずサーバーサイドで) + if (data.reply && data.channel && data.reply.channelId !== data.channel.id) { + if (data.reply.channelId) { + data.channel = await this.channelsRepository.findOneBy({ id: data.reply.channelId }); + } else { + data.channel = null; + } + } + + if (data.isNoteInHanaMode) { + throw new IdentifiableError('cb4feb26-a6e8-44b4-8c9d-d21c48a73d93', 'Unable to re-import notes created in HanaMisskey.'); + } else { + data.isNoteInHanaMode = false; + } + + // チャンネル内にリプライしたら対象のスコープに合わせる + // (クライアントサイドでやっても良い処理だと思うけどとりあえずサーバーサイドで) + if (data.reply && (data.channel == null) && data.reply.channelId) { + data.channel = await this.channelsRepository.findOneBy({ id: data.reply.channelId }); + } + + if (data.createdAt == null) data.createdAt = new Date(); + if (data.visibility == null) data.visibility = 'public'; + if (data.localOnly == null) data.localOnly = false; + if (data.channel != null) data.visibility = 'public'; + if (data.channel != null) data.visibleUsers = []; + if (data.channel != null) data.localOnly = true; + + const meta = await this.metaService.fetch(); + + if (data.visibility === 'public' && data.channel == null) { + const sensitiveWords = meta.sensitiveWords; + if (this.utilityService.isKeyWordIncluded(data.cw ?? data.text ?? '', sensitiveWords)) { + data.visibility = 'home'; + } else if ((await this.roleService.getUserPolicies(user.id)).canPublicNote === false) { + data.visibility = 'home'; + } + } + + const hasProhibitedWords = await this.checkProhibitedWordsContain({ + cw: data.cw, + text: data.text, + pollChoices: data.poll?.choices, + }, meta.prohibitedWords); + + if (hasProhibitedWords) { + throw new IdentifiableError('689ee33f-f97c-479a-ac49-1b9f8140af99', 'Note contains prohibited words'); + } + + const inSilencedInstance = this.utilityService.isSilencedHost(meta.silencedHosts, user.host); + + if (data.visibility === 'public' && inSilencedInstance && user.host !== null) { + data.visibility = 'home'; + } + + if (data.renote) { + switch (data.renote.visibility) { + case 'public': + // public noteは無条件にrenote可能 + break; + case 'home': + // home noteはhome以下にrenote可能 + if (data.visibility === 'public') { + data.visibility = 'home'; + } + break; + case 'followers': + // 他人のfollowers noteはreject + if (data.renote.userId !== user.id) { + throw new Error('Renote target is not public or home'); + } + + // Renote対象がfollowersならfollowersにする + data.visibility = 'followers'; + break; + case 'specified': + // specified / direct noteはreject + throw new Error('Renote target is not public or home'); + } + } + + // Check blocking + if (data.renote && data.text == null && data.poll == null && (data.files == null || data.files.length === 0)) { + if (data.renote.userHost === null) { + if (data.renote.userId !== user.id) { + const blocked = await this.userBlockingService.checkBlocked(data.renote.userId, user.id); + if (blocked) { + throw new Error('blocked'); + } + } + } + } + + // 返信対象がpublicではないならhomeにする + if (data.reply && data.reply.visibility !== 'public' && data.visibility === 'public') { + data.visibility = 'home'; + } + + // ローカルのみをRenoteしたらローカルのみにする + if (data.renote && data.renote.localOnly && data.channel == null) { + data.localOnly = true; + } + + // ローカルのみにリプライしたらローカルのみにする + if (data.reply && data.reply.localOnly && data.channel == null) { + data.localOnly = true; + } + + if (data.text) { + if (data.text.length > DB_MAX_NOTE_TEXT_LENGTH) { + data.text = data.text.slice(0, DB_MAX_NOTE_TEXT_LENGTH); + } + data.text = data.text.trim(); + } else { + data.text = null; + } + + let tags = data.apHashtags; + let emojis = data.apEmojis; + let mentionedUsers = data.apMentions; + + // Parse MFM if needed + if (!tags || !emojis || !mentionedUsers) { + const tokens = (data.text ? mfm.parse(data.text)! : []); + const cwTokens = data.cw ? mfm.parse(data.cw)! : []; + const choiceTokens = data.poll && data.poll.choices + ? concat(data.poll.choices.map(choice => mfm.parse(choice)!)) + : []; + + const combinedTokens = tokens.concat(cwTokens).concat(choiceTokens); + + tags = data.apHashtags ?? extractHashtags(combinedTokens); + + emojis = data.apEmojis ?? extractCustomEmojisFromMfm(combinedTokens); + + mentionedUsers = data.apMentions ?? await this.extractMentionedUsers(user, combinedTokens); + } + + tags = tags.filter(tag => Array.from(tag).length <= 128).splice(0, 32); + + if (data.reply && (user.id !== data.reply.userId) && !mentionedUsers.some(u => u.id === data.reply!.userId)) { + mentionedUsers.push(await this.usersRepository.findOneByOrFail({ id: data.reply!.userId })); + } + + if (data.visibility === 'specified') { + if (data.visibleUsers == null) throw new Error('invalid param'); + + for (const u of data.visibleUsers) { + if (!mentionedUsers.some(x => x.id === u.id)) { + mentionedUsers.push(u); + } + } + + if (data.reply && !data.visibleUsers.some(x => x.id === data.reply!.userId)) { + data.visibleUsers.push(await this.usersRepository.findOneByOrFail({ id: data.reply!.userId })); + } + } + + const note = await this.insertNote(user, data, tags, emojis, mentionedUsers); + + setImmediate('post created', { signal: this.#shutdownController.signal }).then( + () => this.postNoteImported(note, user, data, silent, tags!, mentionedUsers!), + () => { /* aborted, ignore this */ }, + ); + + return note; + } + @bindThis private async insertNote(user: { id: MiUser['id']; host: MiUser['host']; }, data: Option, tags: string[], emojis: string[], mentionedUsers: MinimumUser[]) { + if (data.createdAt) { + if (data.createdAt.getTime() > Date.now() + 1000 * 60 * 3 ) { + throw new Error('Invalid createdAt time: Time is more than 3 minutes ahead of the current time.'); + } + } + const insert = new MiNote({ id: this.idService.gen(data.createdAt?.getTime()), fileIds: data.files ? data.files.map(file => file.id) : [], @@ -424,6 +613,7 @@ export class NoteCreateService implements OnApplicationShutdown { emojis, userId: user.id, localOnly: data.localOnly!, + isNoteInHanaMode: data.isNoteInHanaMode!, reactionAcceptance: data.reactionAcceptance, visibility: data.visibility as any, visibleUserIds: data.visibility === 'specified' @@ -533,7 +723,13 @@ export class NoteCreateService implements OnApplicationShutdown { // Increment notes count (user) this.incNotesCountOfUser(user); - this.pushToTl(note, user); + // はなモードが有効なユーザーであることと、はなモード内でのノートであることは等価であることが保証されているので + // チャンネルに関してもこれでOK + if (note.isNoteInHanaMode) { + this.pushToTl(note, user, ['localTimeline']); + } else { + this.pushToTl(note, user); + }; this.antennaService.addNoteToAntennas(note, user); @@ -738,6 +934,104 @@ export class NoteCreateService implements OnApplicationShutdown { this.index(note); } + @bindThis + private async postNoteImported(note: MiNote, user: { + id: MiUser['id']; + username: MiUser['username']; + host: MiUser['host']; + isBot: MiUser['isBot']; + }, data: Option, silent: boolean, tags: string[], mentionedUsers: MinimumUser[]) { + const meta = await this.metaService.fetch(); + + this.notesChart.update(note, true); + if (note.visibility !== 'specified' && (meta.enableChartsForRemoteUser || (user.host == null))) { + this.perUserNotesChart.update(user, note, true); + } + + // Register host + if (this.userEntityService.isRemoteUser(user)) { + this.federatedInstanceService.fetch(user.host).then(async i => { + if (note.renote && note.text) { + this.instancesRepository.increment({ id: i.id }, 'notesCount', 1); + } else if (!note.renote) { + this.instancesRepository.increment({ id: i.id }, 'notesCount', 1); + } + if ((await this.metaService.fetch()).enableChartsForFederatedInstances) { + this.instanceChart.updateNote(i.host, note, true); + } + }); + } + + if (data.renote && data.text) { + // Increment notes count (user) + this.incNotesCountOfUser(user); + } else if (!data.renote) { + // Increment notes count (user) + this.incNotesCountOfUser(user); + } + + this.pushToTl(note, user, ['localTimeline', 'homeTimeline', 'userListTimeline', 'antennaTimeline']); + + this.antennaService.addNoteToAntennas(note, user); + + if (data.reply) { + this.saveReply(data.reply, note); + } + + if (data.reply == null) { + // TODO: キャッシュ + this.followingsRepository.findBy({ + followeeId: user.id, + notify: 'normal', + }).then(followings => { + for (const following of followings) { + // TODO: ワードミュート考慮 + this.notificationService.createNotification(following.followerId, 'note', { + noteId: note.id, + }, user.id); + } + }); + } + + if (data.renote && data.text == null && data.renote.userId !== user.id && !user.isBot) { + this.incRenoteCount(data.renote); + } + + if (data.poll && data.poll.expiresAt) { + const delay = data.poll.expiresAt.getTime() - Date.now(); + this.queueService.endedPollNotificationQueue.add(note.id, { + noteId: note.id, + }, { + delay, + removeOnComplete: true, + }); + } + + // Pack the note + const noteObj = await this.noteEntityService.pack(note, null, { skipHide: true, withReactionAndUserPairCache: true }); + + if (data.channel) { + this.channelsRepository.increment({ id: data.channel.id }, 'notesCount', 1); + this.channelsRepository.update(data.channel.id, { + lastNotedAt: new Date(), + }); + + this.notesRepository.countBy({ + userId: user.id, + channelId: data.channel.id, + }).then(count => { + // この処理が行われるのはノート作成後なので、ノートが一つしかなかったら最初の投稿だと判断できる + // TODO: とはいえノートを削除して何回も投稿すればその分だけインクリメントされる雑さもあるのでどうにかしたい + if (count === 1) { + this.channelsRepository.increment({ id: data.channel!.id }, 'usersCount', 1); + } + }); + } + + // Register to search database + this.index(note); + } + @bindThis private isRenote(note: Option): note is Option & { renote: MiNote } { return note.renote != null; @@ -764,16 +1058,18 @@ export class NoteCreateService implements OnApplicationShutdown { .where('id = :id', { id: renote.id }) .execute(); - // 30%の確率、3日以内に投稿されたノートの場合ハイライト用ランキング更新 - if (Math.random() < 0.3 && (Date.now() - this.idService.parse(renote.id).date.getTime()) < 1000 * 60 * 60 * 24 * 3) { + // 3日以内に投稿されたノートの場合ハイライト用ランキング更新 + if ((Date.now() - this.idService.parse(renote.id).date.getTime()) < 1000 * 60 * 60 * 24 * 3) { if (renote.channelId != null) { if (renote.replyId == null) { this.featuredService.updateInChannelNotesRanking(renote.channelId, renote.id, 5); } } else { - if (renote.visibility === 'public' && renote.userHost == null && renote.replyId == null) { + if (renote.visibility === 'public' && renote.replyId == null) { this.featuredService.updateGlobalNotesRanking(renote.id, 5); - this.featuredService.updatePerUserNotesRanking(renote.userId, renote.id, 5); + if (renote.userHost == null) { + this.featuredService.updatePerUserNotesRanking(renote.userId, renote.id, 5); + } } } } @@ -863,16 +1159,25 @@ export class NoteCreateService implements OnApplicationShutdown { } @bindThis - private async pushToTl(note: MiNote, user: { id: MiUser['id']; host: MiUser['host']; }) { + private async pushToTl(note: MiNote, user: { id: MiUser['id']; host: MiUser['host']; }, notToPush?: FanoutTimelineNamePrefix[]) { const meta = await this.metaService.fetch(); if (!meta.enableFanoutTimeline) return; const r = this.redisForTimelines.pipeline(); + const notToPushSet = notToPush ? new Set(notToPush) : null; + const shouldPush = (prefix: FanoutTimelineNamePrefix): boolean => { + return !notToPushSet || !notToPushSet.has(prefix); + }; + if (note.channelId) { - this.fanoutTimelineService.push(`channelTimeline:${note.channelId}`, note.id, this.config.perChannelMaxNoteCacheCount, r); + if (shouldPush('channelTimeline')) { + this.fanoutTimelineService.push(`channelTimeline:${note.channelId}`, note.id, this.config.perChannelMaxNoteCacheCount, r); + } - this.fanoutTimelineService.push(`userTimelineWithChannel:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r); + if (shouldPush('userTimeline')) { + this.fanoutTimelineService.push(`userTimelineWithChannel:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r); + } const channelFollowings = await this.channelFollowingsRepository.find({ where: { @@ -882,9 +1187,11 @@ export class NoteCreateService implements OnApplicationShutdown { }); for (const channelFollowing of channelFollowings) { - this.fanoutTimelineService.push(`homeTimeline:${channelFollowing.followerId}`, note.id, meta.perUserHomeTimelineCacheMax, r); - if (note.fileIds.length > 0) { - this.fanoutTimelineService.push(`homeTimelineWithFiles:${channelFollowing.followerId}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r); + if (shouldPush('homeTimeline')) { + this.fanoutTimelineService.push(`homeTimeline:${channelFollowing.followerId}`, note.id, meta.perUserHomeTimelineCacheMax, r); + if (note.fileIds.length > 0) { + this.fanoutTimelineService.push(`homeTimelineWithFiles:${channelFollowing.followerId}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r); + } } } } else { @@ -908,7 +1215,6 @@ export class NoteCreateService implements OnApplicationShutdown { ]); if (note.visibility === 'followers') { - // TODO: 重そうだから何とかしたい Set 使う？ userListMemberships = userListMemberships.filter(x => x.userListUserId === user.id || followings.some(f => f.followerId === x.userListUserId)); } @@ -922,61 +1228,73 @@ export class NoteCreateService implements OnApplicationShutdown { if (!following.withReplies) continue; } - this.fanoutTimelineService.push(`homeTimeline:${following.followerId}`, note.id, meta.perUserHomeTimelineCacheMax, r); - if (note.fileIds.length > 0) { - this.fanoutTimelineService.push(`homeTimelineWithFiles:${following.followerId}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r); + if (shouldPush('homeTimeline')) { + this.fanoutTimelineService.push(`homeTimeline:${following.followerId}`, note.id, meta.perUserHomeTimelineCacheMax, r); + if (note.fileIds.length > 0) { + this.fanoutTimelineService.push(`homeTimelineWithFiles:${following.followerId}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r); + } } } for (const userListMembership of userListMemberships) { // ダイレクトのとき、そのリストが対象外のユーザーの場合 - if ( - note.visibility === 'specified' && - note.userId !== userListMembership.userListUserId && - !note.visibleUserIds.some(v => v === userListMembership.userListUserId) - ) continue; - // 「自分自身への返信 or そのリストの作成者への返信」のどちらでもない場合 + if (note.visibility === 'specified' && + note.userId !== userListMembership.userListUserId && + !note.visibleUserIds.some(v => v === userListMembership.userListUserId)) continue; + if (isReply(note, userListMembership.userListUserId)) { if (!userListMembership.withReplies) continue; } - this.fanoutTimelineService.push(`userListTimeline:${userListMembership.userListId}`, note.id, meta.perUserListTimelineCacheMax, r); - if (note.fileIds.length > 0) { - this.fanoutTimelineService.push(`userListTimelineWithFiles:${userListMembership.userListId}`, note.id, meta.perUserListTimelineCacheMax / 2, r); + if (shouldPush('userListTimeline')) { + this.fanoutTimelineService.push(`userListTimeline:${userListMembership.userListId}`, note.id, meta.perUserListTimelineCacheMax, r); + if (note.fileIds.length > 0) { + this.fanoutTimelineService.push(`userListTimelineWithFiles:${userListMembership.userListId}`, note.id, meta.perUserListTimelineCacheMax / 2, r); + } } } // 自分自身のHTL if (note.userHost == null) { if (note.visibility !== 'specified' || !note.visibleUserIds.some(v => v === user.id)) { - this.fanoutTimelineService.push(`homeTimeline:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax, r); - if (note.fileIds.length > 0) { - this.fanoutTimelineService.push(`homeTimelineWithFiles:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r); + if (shouldPush('homeTimeline')) { + this.fanoutTimelineService.push(`homeTimeline:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax, r); + if (note.fileIds.length > 0) { + this.fanoutTimelineService.push(`homeTimelineWithFiles:${user.id}`, note.id, meta.perUserHomeTimelineCacheMax / 2, r); + } } } } // 自分自身以外への返信 if (isReply(note)) { - this.fanoutTimelineService.push(`userTimelineWithReplies:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r); + if (shouldPush('userTimeline')) { + this.fanoutTimelineService.push(`userTimelineWithReplies:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r); + } if (note.visibility === 'public' && note.userHost == null) { - this.fanoutTimelineService.push('localTimelineWithReplies', note.id, 300, r); - if (note.replyUserHost == null) { - this.fanoutTimelineService.push(`localTimelineWithReplyTo:${note.replyUserId}`, note.id, 300 / 10, r); + if (shouldPush('localTimeline')) { + this.fanoutTimelineService.push('localTimelineWithReplies', note.id, 300, r); + if (note.replyUserHost == null) { + this.fanoutTimelineService.push(`localTimelineWithReplyTo:${note.replyUserId}`, note.id, 300 / 10, r); + } } } } else { - this.fanoutTimelineService.push(`userTimeline:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r); - if (note.fileIds.length > 0) { - this.fanoutTimelineService.push(`userTimelineWithFiles:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax / 2 : meta.perRemoteUserUserTimelineCacheMax / 2, r); + if (shouldPush('userTimeline')) { + this.fanoutTimelineService.push(`userTimeline:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax : meta.perRemoteUserUserTimelineCacheMax, r); + if (note.fileIds.length > 0) { + this.fanoutTimelineService.push(`userTimelineWithFiles:${user.id}`, note.id, note.userHost == null ? meta.perLocalUserUserTimelineCacheMax / 2 : meta.perRemoteUserUserTimelineCacheMax / 2, r); + } } if (note.visibility === 'public' && note.userHost == null) { - this.fanoutTimelineService.push('localTimeline', note.id, 1000, r); - if (note.fileIds.length > 0) { - this.fanoutTimelineService.push('localTimelineWithFiles', note.id, 500, r); + if (shouldPush('localTimeline')) { + this.fanoutTimelineService.push('localTimeline', note.id, 1000, r); + if (note.fileIds.length > 0) { + this.fanoutTimelineService.push('localTimelineWithFiles', note.id, 500, r); + } } } } diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts index 18a006986756..d815eba97a09 100644 --- a/packages/backend/src/core/QueueService.ts +++ b/packages/backend/src/core/QueueService.ts @@ -36,6 +36,7 @@ import type { } from './QueueModule.js'; import type httpSignature from '@peertube/http-signature'; import type * as Bull from 'bullmq'; +import { MiNote } from '@/models/Note.js'; @Injectable() export class QueueService { @@ -287,6 +288,54 @@ export class QueueService { }); } + @bindThis + public createImportNotesJob(user: ThinUser, fileId: MiDriveFile['id'], type: string | null | undefined) { + return this.dbQueue.add('importNotes', { + user: { id: user.id }, + fileId: fileId, + type: type, + }, { + removeOnComplete: true, + removeOnFail: true, + }); + } + + @bindThis + public createImportTweetsToDbJob(user: ThinUser, targets: string[], note: MiNote['id'] | null) { + const jobs = targets.map(rel => this.generateToDbJobData('importTweetsToDb', { user, target: rel, note })); + return this.dbQueue.addBulk(jobs); + } + + @bindThis + public createImportMastoToDbJob(user: ThinUser, targets: string[], note: MiNote['id'] | null) { + const jobs = targets.map(rel => this.generateToDbJobData('importMastoToDb', { user, target: rel, note })); + return this.dbQueue.addBulk(jobs); + } + + @bindThis + public createImportPleroToDbJob(user: ThinUser, targets: string[], note: MiNote['id'] | null) { + const jobs = targets.map(rel => this.generateToDbJobData('importPleroToDb', { user, target: rel, note })); + return this.dbQueue.addBulk(jobs); + } + + @bindThis + public createImportKeyNotesToDbJob(user: ThinUser, targets: string[], note: MiNote['id'] | null) { + const jobs = targets.map(rel => this.generateToDbJobData('importKeyNotesToDb', { user, target: rel, note })); + return this.dbQueue.addBulk(jobs); + } + + @bindThis + public createImportIGToDbJob(user: ThinUser, targets: string[]) { + const jobs = targets.map(rel => this.generateToDbJobData('importIGToDb', { user, target: rel })); + return this.dbQueue.addBulk(jobs); + } + + @bindThis + public createImportFBToDbJob(user: ThinUser, targets: string[]) { + const jobs = targets.map(rel => this.generateToDbJobData('importFBToDb', { user, target: rel })); + return this.dbQueue.addBulk(jobs); + } + @bindThis public createImportFollowingToDbJob(user: ThinUser, targets: string[], withReplies?: boolean) { const jobs = targets.map(rel => this.generateToDbJobData('importFollowingToDb', { user, target: rel, withReplies })); @@ -322,7 +371,7 @@ export class QueueService { } @bindThis - private generateToDbJobData>(name: T, data: D): { + private generateToDbJobData>(name: T, data: D): { name: string, data: D, opts: Bull.JobsOptions, diff --git a/packages/backend/src/core/ReactionService.ts b/packages/backend/src/core/ReactionService.ts index 371207c33a7d..2fde6cb3c5d9 100644 --- a/packages/backend/src/core/ReactionService.ts +++ b/packages/backend/src/core/ReactionService.ts @@ -209,9 +209,8 @@ export class ReactionService { .where('id = :id', { id: note.id }) .execute(); - // 30%の確率、セルフではない、3日以内に投稿されたノートの場合ハイライト用ランキング更新 + // セルフではない、3日以内に投稿されたノートの場合ハイライト用ランキング更新 if ( - Math.random() < 0.3 && note.userId !== user.id && (Date.now() - this.idService.parse(note.id).date.getTime()) < 1000 * 60 * 60 * 24 * 3 ) { @@ -220,9 +219,11 @@ export class ReactionService { this.featuredService.updateInChannelNotesRanking(note.channelId, note.id, 1); } } else { - if (note.visibility === 'public' && note.userHost == null && note.replyId == null) { + if (note.visibility === 'public' && note.replyId == null) { this.featuredService.updateGlobalNotesRanking(note.id, 1); - this.featuredService.updatePerUserNotesRanking(note.userId, note.id, 1); + if (note.userHost == null) { + this.featuredService.updatePerUserNotesRanking(note.userId, note.id, 1); + } } } } diff --git a/packages/backend/src/core/RoleService.ts b/packages/backend/src/core/RoleService.ts index 796677467364..4a6039d78af5 100644 --- a/packages/backend/src/core/RoleService.ts +++ b/packages/backend/src/core/RoleService.ts @@ -34,6 +34,7 @@ import type { OnApplicationShutdown, OnModuleInit } from '@nestjs/common'; export type RolePolicies = { gtlAvailable: boolean; ltlAvailable: boolean; + hanamiTlAvailable: boolean; canPublicNote: boolean; mentionLimit: number; canInvite: boolean; @@ -58,11 +59,13 @@ export type RolePolicies = { userEachUserListsLimit: number; rateLimitFactor: number; avatarDecorationLimit: number; + canImportNotes: boolean; }; export const DEFAULT_POLICIES: RolePolicies = { gtlAvailable: true, ltlAvailable: true, + hanamiTlAvailable: true, canPublicNote: true, mentionLimit: 20, canInvite: false, @@ -87,6 +90,7 @@ export const DEFAULT_POLICIES: RolePolicies = { userEachUserListsLimit: 50, rateLimitFactor: 1, avatarDecorationLimit: 1, + canImportNotes: true, }; @Injectable() @@ -247,6 +251,10 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { case 'isCat': { return user.isCat; } + // はなモードが有効 + case 'isInHanaMode':{ + return user.isInHanaMode; + } // 「ユーザを見つけやすくする」が有効なアカウント case 'isExplorable': { return user.isExplorable; @@ -365,6 +373,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { return { gtlAvailable: calc('gtlAvailable', vs => vs.some(v => v === true)), ltlAvailable: calc('ltlAvailable', vs => vs.some(v => v === true)), + hanamiTlAvailable: calc('hanamiTlAvailable', vs => vs.some(v => v === true)), canPublicNote: calc('canPublicNote', vs => vs.some(v => v === true)), mentionLimit: calc('mentionLimit', vs => Math.max(...vs)), canInvite: calc('canInvite', vs => vs.some(v => v === true)), @@ -389,6 +398,7 @@ export class RoleService implements OnApplicationShutdown, OnModuleInit { userEachUserListsLimit: calc('userEachUserListsLimit', vs => Math.max(...vs)), rateLimitFactor: calc('rateLimitFactor', vs => Math.max(...vs)), avatarDecorationLimit: calc('avatarDecorationLimit', vs => Math.max(...vs)), + canImportNotes: calc('canImportNotes', vs => vs.some(v => v === true)), }; } diff --git a/packages/backend/src/core/activitypub/models/ApNoteService.ts b/packages/backend/src/core/activitypub/models/ApNoteService.ts index 5b75da22a033..360c47014297 100644 --- a/packages/backend/src/core/activitypub/models/ApNoteService.ts +++ b/packages/backend/src/core/activitypub/models/ApNoteService.ts @@ -303,6 +303,7 @@ export class ApNoteService { cw, text, localOnly: false, + isNoteInHanaMode: false, visibility, visibleUsers, apMentions, diff --git a/packages/backend/src/core/entities/MetaEntityService.ts b/packages/backend/src/core/entities/MetaEntityService.ts index 4ab11ccbb981..436bd4fb0110 100644 --- a/packages/backend/src/core/entities/MetaEntityService.ts +++ b/packages/backend/src/core/entities/MetaEntityService.ts @@ -156,6 +156,7 @@ export class MetaEntityService { proxyAccountName: proxyAccount ? proxyAccount.username : null, features: { localTimeline: instance.policies.ltlAvailable, + hanamiTimeline: instance.policies.hanamiTlAvailable, globalTimeline: instance.policies.gtlAvailable, registration: !instance.disableRegistration, emailRequiredForSignup: instance.emailRequiredForSignup, diff --git a/packages/backend/src/core/entities/NoteEntityService.ts b/packages/backend/src/core/entities/NoteEntityService.ts index d121f78bf4ec..69f8374d88f7 100644 --- a/packages/backend/src/core/entities/NoteEntityService.ts +++ b/packages/backend/src/core/entities/NoteEntityService.ts @@ -358,7 +358,7 @@ export class NoteEntityService implements OnModuleInit { url: note.url ?? undefined, poll: note.hasPoll ? this.populatePoll(note, meId) : undefined, deleteAt: note.deleteAt?.toISOString() ?? undefined, - + isNoteInHanaMode: note.isNoteInHanaMode, ...(opts.detail ? { clippedCount: note.clippedCount, diff --git a/packages/backend/src/core/entities/UserEntityService.ts b/packages/backend/src/core/entities/UserEntityService.ts index 2c9226f91475..b58cde1b31cd 100644 --- a/packages/backend/src/core/entities/UserEntityService.ts +++ b/packages/backend/src/core/entities/UserEntityService.ts @@ -492,6 +492,7 @@ export class UserEntityService implements OnModuleInit { }))) : [], isBot: user.isBot, isCat: user.isCat, + isInHanaMode: user.isInHanaMode, instance: user.host ? this.federatedInstanceService.federatedInstanceCache.fetch(user.host).then(instance => instance ? { name: instance.name, softwareName: instance.softwareName, @@ -510,7 +511,7 @@ export class UserEntityService implements OnModuleInit { name: r.name, iconUrl: r.iconUrl, displayOrder: r.displayOrder, - })) + })), ) : undefined, ...(isDetailed ? { diff --git a/packages/backend/src/misc/create-temp.ts b/packages/backend/src/misc/create-temp.ts index 9aaecf826346..6cc896046fb7 100644 --- a/packages/backend/src/misc/create-temp.ts +++ b/packages/backend/src/misc/create-temp.ts @@ -9,7 +9,7 @@ export function createTemp(): Promise<[string, () => void]> { return new Promise<[string, () => void]>((res, rej) => { tmp.file((e, path, fd, cleanup) => { if (e) return rej(e); - res([path, process.env.NODE_ENV === 'production' ? cleanup : () => {}]); + res([path, process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'development' ? cleanup : () => {}]); }); }); } @@ -22,7 +22,7 @@ export function createTempDir(): Promise<[string, () => void]> { }, (e, path, cleanup) => { if (e) return rej(e); - res([path, process.env.NODE_ENV === 'production' ? cleanup : () => {}]); + res([path, process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'development' ? cleanup : () => {}]); }, ); }); diff --git a/packages/backend/src/models/Note.ts b/packages/backend/src/models/Note.ts index c25f46be4314..5560da53ae8a 100644 --- a/packages/backend/src/models/Note.ts +++ b/packages/backend/src/models/Note.ts @@ -83,6 +83,11 @@ export class MiNote { }) public localOnly: boolean; + @Column('boolean', { + default: false, + }) + public isNoteInHanaMode: boolean; + @Column('varchar', { length: 64, nullable: true, }) diff --git a/packages/backend/src/models/Role.ts b/packages/backend/src/models/Role.ts index a173971b2ce2..3079c1ff13a4 100644 --- a/packages/backend/src/models/Role.ts +++ b/packages/backend/src/models/Role.ts @@ -83,6 +83,13 @@ type CondFormulaValueIsCat = { type: 'isCat'; }; +/** + * はなモードを有効にしたアカウントの場合のみ成立とする + */ +type CondFormulaValueIsinHanaMode = { + type: 'isInHanaMode'; +}; + /** * 「ユーザを見つけやすくする」が有効なアカウントの場合のみ成立とする */ @@ -164,6 +171,7 @@ export type RoleCondFormulaValue = { id: string } & ( CondFormulaValueIsLocked | CondFormulaValueIsBot | CondFormulaValueIsCat | + CondFormulaValueIsinHanaMode | CondFormulaValueIsExplorable | CondFormulaValueRoleAssignedTo | CondFormulaValueCreatedLessThan | diff --git a/packages/backend/src/models/User.ts b/packages/backend/src/models/User.ts index 9e2d7a34447d..0fcdb50e2a79 100644 --- a/packages/backend/src/models/User.ts +++ b/packages/backend/src/models/User.ts @@ -179,6 +179,12 @@ export class MiUser { }) public isCat: boolean; + @Column('boolean', { + default: false, + comment: 'Whether the User is in Hana Mode.', + }) + public isInHanaMode: boolean; + @Column('boolean', { default: false, comment: 'Whether the User is the root.', diff --git a/packages/backend/src/models/json-schema/meta.ts b/packages/backend/src/models/json-schema/meta.ts index 07c725871cff..cfd27b164a40 100644 --- a/packages/backend/src/models/json-schema/meta.ts +++ b/packages/backend/src/models/json-schema/meta.ts @@ -284,6 +284,10 @@ export const packedMetaDetailedOnlySchema = { type: 'boolean', optional: false, nullable: false, }, + hanamiTimeline: { + type: 'boolean', + optional: false, nullable: false, + }, hcaptcha: { type: 'boolean', optional: false, nullable: false, diff --git a/packages/backend/src/models/json-schema/note.ts b/packages/backend/src/models/json-schema/note.ts index 432c096e484c..9f12827c3bea 100644 --- a/packages/backend/src/models/json-schema/note.ts +++ b/packages/backend/src/models/json-schema/note.ts @@ -201,6 +201,10 @@ export const packedNoteSchema = { type: 'boolean', optional: true, nullable: false, }, + isNoteInHanaMode: { + type: 'boolean', + optional: true, nullable: false, + }, reactionAcceptance: { type: 'string', optional: false, nullable: true, diff --git a/packages/backend/src/models/json-schema/role.ts b/packages/backend/src/models/json-schema/role.ts index 7366f053560d..184f44ea9344 100644 --- a/packages/backend/src/models/json-schema/role.ts +++ b/packages/backend/src/models/json-schema/role.ts @@ -3,6 +3,8 @@ * SPDX-License-Identifier: AGPL-3.0-only */ +import { Optional } from '@nestjs/common'; + export const packedRoleCondFormulaLogicsSchema = { type: 'object', properties: { @@ -176,6 +178,10 @@ export const packedRolePoliciesSchema = { type: 'boolean', optional: false, nullable: false, }, + hanamiTlAvailable: { + type: 'boolean', + optional: false, nullable: false, + }, canPublicNote: { type: 'boolean', optional: false, nullable: false, @@ -188,6 +194,10 @@ export const packedRolePoliciesSchema = { type: 'boolean', optional: false, nullable: false, }, + canImportNotes: { + type: 'boolean', + optional: false, nullable: false, + }, inviteLimit: { type: 'integer', optional: false, nullable: false, diff --git a/packages/backend/src/models/json-schema/user.ts b/packages/backend/src/models/json-schema/user.ts index 947a9317d7a6..a2454e89d9d0 100644 --- a/packages/backend/src/models/json-schema/user.ts +++ b/packages/backend/src/models/json-schema/user.ts @@ -115,6 +115,10 @@ export const packedUserLiteSchema = { type: 'boolean', nullable: false, optional: true, }, + isInHanaMode: { + type: 'boolean', + nullable: false, optional: true, + }, instance: { type: 'object', nullable: false, optional: true, diff --git a/packages/backend/src/queue/QueueProcessorModule.ts b/packages/backend/src/queue/QueueProcessorModule.ts index 3d0e398fd3e1..01d2a6bfaf4e 100644 --- a/packages/backend/src/queue/QueueProcessorModule.ts +++ b/packages/backend/src/queue/QueueProcessorModule.ts @@ -31,6 +31,7 @@ import { ExportUserListsProcessorService } from './processors/ExportUserListsPro import { ExportAntennasProcessorService } from './processors/ExportAntennasProcessorService.js'; import { ImportBlockingProcessorService } from './processors/ImportBlockingProcessorService.js'; import { ImportCustomEmojisProcessorService } from './processors/ImportCustomEmojisProcessorService.js'; +import { ImportNotesProcessorService } from './processors/ImportNotesProcessorService.js'; import { ImportFollowingProcessorService } from './processors/ImportFollowingProcessorService.js'; import { ImportMutingProcessorService } from './processors/ImportMutingProcessorService.js'; import { ImportUserListsProcessorService } from './processors/ImportUserListsProcessorService.js'; @@ -63,6 +64,7 @@ import { RelationshipProcessorService } from './processors/RelationshipProcessor ExportBlockingProcessorService, ExportUserListsProcessorService, ExportAntennasProcessorService, + ImportNotesProcessorService, ImportFollowingProcessorService, ImportMutingProcessorService, ImportBlockingProcessorService, diff --git a/packages/backend/src/queue/QueueProcessorService.ts b/packages/backend/src/queue/QueueProcessorService.ts index bc16a422fcac..4ffa0512110c 100644 --- a/packages/backend/src/queue/QueueProcessorService.ts +++ b/packages/backend/src/queue/QueueProcessorService.ts @@ -44,6 +44,7 @@ import { CleanProcessorService } from './processors/CleanProcessorService.js'; import { AggregateRetentionProcessorService } from './processors/AggregateRetentionProcessorService.js'; import { QueueLoggerService } from './QueueLoggerService.js'; import { QUEUE, baseQueueOptions } from './const.js'; +import { ImportNotesProcessorService } from './processors/ImportNotesProcessorService.js'; // ref. https://github.com/misskey-dev/misskey/pull/7635#issue-971097019 function httpRelatedBackoff(attemptsMade: number) { @@ -108,6 +109,7 @@ export class QueueProcessorService implements OnApplicationShutdown { private exportUserListsProcessorService: ExportUserListsProcessorService, private exportAntennasProcessorService: ExportAntennasProcessorService, private importFollowingProcessorService: ImportFollowingProcessorService, + private importNotesProcessorService: ImportNotesProcessorService, private importMutingProcessorService: ImportMutingProcessorService, private importBlockingProcessorService: ImportBlockingProcessorService, private importUserListsProcessorService: ImportUserListsProcessorService, @@ -202,6 +204,12 @@ export class QueueProcessorService implements OnApplicationShutdown { case 'exportAntennas': return this.exportAntennasProcessorService.process(job); case 'importFollowing': return this.importFollowingProcessorService.process(job); case 'importFollowingToDb': return this.importFollowingProcessorService.processDb(job); + case 'importNotes': return this.importNotesProcessorService.process(job); + case 'importTweetsToDb': return this.importNotesProcessorService.processTwitterDb(job); + case 'importIGToDb': return this.importNotesProcessorService.processIGDb(job); + case 'importMastoToDb': return this.importNotesProcessorService.processMastoToDb(job); + case 'importPleroToDb': return this.importNotesProcessorService.processPleroToDb(job); + case 'importKeyNotesToDb': return this.importNotesProcessorService.processKeyNotesToDb(job); case 'importMuting': return this.importMutingProcessorService.process(job); case 'importBlocking': return this.importBlockingProcessorService.process(job); case 'importBlockingToDb': return this.importBlockingProcessorService.processDb(job); diff --git a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts new file mode 100644 index 000000000000..f6b745768ba7 --- /dev/null +++ b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts @@ -0,0 +1,717 @@ +import * as fs from 'node:fs'; +import * as fsp from 'node:fs/promises'; +import * as crypto from 'node:crypto'; +import { Inject, Injectable } from '@nestjs/common'; +import { ZipReader } from 'slacc'; +import { DI } from '@/di-symbols.js'; +import type { UsersRepository, DriveFilesRepository, MiDriveFile, MiNote, NotesRepository, MiUser, DriveFoldersRepository, MiDriveFolder } from '@/models/_.js'; +import type Logger from '@/logger.js'; +import { DownloadService } from '@/core/DownloadService.js'; +import { bindThis } from '@/decorators.js'; +import { QueueService } from '@/core/QueueService.js'; +import { createTemp, createTempDir } from '@/misc/create-temp.js'; +import { NoteCreateService } from '@/core/NoteCreateService.js'; +import { DriveService } from '@/core/DriveService.js'; +import { MfmService } from '@/core/MfmService.js'; +import { ApNoteService } from '@/core/activitypub/models/ApNoteService.js'; +import { extractApHashtagObjects } from '@/core/activitypub/models/tag.js'; +import { IdService } from '@/core/IdService.js'; +import type { Config } from '@/config.js'; +import { QueueLoggerService } from '../QueueLoggerService.js'; +import type * as Bull from 'bullmq'; +import type { DbNoteImportToDbJobData, DbNoteImportJobData, DbNoteWithParentImportToDbJobData } from '../types.js'; + +@Injectable() +export class ImportNotesProcessorService { + private logger: Logger; + + constructor( + @Inject(DI.config) + private config: Config, + + @Inject(DI.usersRepository) + private usersRepository: UsersRepository, + + @Inject(DI.driveFilesRepository) + private driveFilesRepository: DriveFilesRepository, + + @Inject(DI.driveFoldersRepository) + private driveFoldersRepository: DriveFoldersRepository, + + @Inject(DI.notesRepository) + private notesRepository: NotesRepository, + + private queueService: QueueService, + private noteCreateService: NoteCreateService, + private mfmService: MfmService, + private apNoteService: ApNoteService, + private driveService: DriveService, + private downloadService: DownloadService, + private idService: IdService, + private queueLoggerService: QueueLoggerService, + ) { + this.logger = this.queueLoggerService.logger.createSubLogger('import-notes'); + } + + @bindThis + private async uploadFiles(dir: string, user: MiUser, folder?: MiDriveFolder['id']) { + const fileList = await fsp.readdir(dir); + for await (const file of fileList) { + const name = `${dir}/${file}`; + if (fs.statSync(name).isDirectory()) { + await this.uploadFiles(name, user, folder); + } else { + const exists = await this.driveFilesRepository.findOneBy({ name: file, userId: user.id, folderId: folder }); + + if (file.endsWith('.srt')) return; + + if (!exists) { + await this.driveService.addFile({ + user: user, + path: name, + name: file, + folderId: folder, + }); + } + } + } + } + + @bindThis + private downloadUrl(url: string, path:string): Promise<{filename: string}> { + return this.downloadService.downloadUrl(url, path, { operationTimeout: this.config.import?.downloadTimeout, maxSize: this.config.import?.maxFileSize }); + } + + @bindThis + private async recreateChain(idFieldPath: string[], replyFieldPath: string[], arr: any[], includeOrphans: boolean): Promise { + type NotesMap = { + [id: string]: any; + }; + const notesTree: any[] = []; + const noteById: NotesMap = {}; + const notesWaitingForParent: NotesMap = {}; + + for await (const note of arr) { + const noteId = idFieldPath.reduce( + (obj, step) => obj[step], + note, + ); + + noteById[noteId] = note; + note.childNotes = []; + + const children = notesWaitingForParent[noteId]; + if (children) { + note.childNotes.push(...children); + delete notesWaitingForParent[noteId]; + } + + const noteReplyId = replyFieldPath.reduce( + (obj, step) => obj[step], + note, + ); + if (noteReplyId == null) { + notesTree.push(note); + continue; + } + + const parent = noteById[noteReplyId]; + if (parent) { + parent.childNotes.push(note); + } else { + notesWaitingForParent[noteReplyId] ||= []; + notesWaitingForParent[noteReplyId].push(note); + } + } + + if (includeOrphans) { + notesTree.push(...Object.values(notesWaitingForParent).flat(1)); + } + + return notesTree; + } + + @bindThis + private isIterable(obj: any) { + if (obj == null) { + return false; + } + return typeof obj[Symbol.iterator] === 'function'; + } + + @bindThis + private parseTwitterFile(str : string) : { tweet: object }[] { + const jsonStr = str.replace(/^\s*window\.YTD\.tweets\.part0\s*=\s*/, ''); + + try { + return JSON.parse(jsonStr); + } catch (error) { + //The format is not what we expected. Either this file was tampered with or twitters exports changed + this.logger.warn('Failed to import twitter notes due to malformed file'); + throw error; + } + } + + @bindThis + public async process(job: Bull.Job): Promise { + this.logger.info(`Starting note import of ${job.data.user.id} ...`); + + const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); + if (user == null) { + return; + } + + const file = await this.driveFilesRepository.findOneBy({ + id: job.data.fileId, + }); + if (file == null) { + return; + } + + let folder = await this.driveFoldersRepository.findOneBy({ name: 'Imports', userId: job.data.user.id }); + if (folder == null) { + await this.driveFoldersRepository.insert({ id: this.idService.gen(), name: 'Imports', userId: job.data.user.id }); + folder = await this.driveFoldersRepository.findOneBy({ name: 'Imports', userId: job.data.user.id }); + } + + const type = job.data.type; + + if (type === 'Twitter' || file.name.startsWith('twitter') && file.name.endsWith('.zip')) { + const [path, cleanup] = await createTempDir(); + + this.logger.info(`Temp dir is ${path}`); + + const destPath = path + '/twitter.zip'; + + try { + await fsp.writeFile(destPath, '', 'binary'); + await this.downloadUrl(file.url, destPath); + } catch (e) { // TODO: 何度か再試行 + if (e instanceof Error || typeof e === 'string') { + this.logger.error(e); + } + throw e; + } + + const outputPath = path + '/twitter'; + try { + this.logger.succ(`Unzipping to ${outputPath}`); + ZipReader.withDestinationPath(outputPath).viaBuffer(await fsp.readFile(destPath)); + + const unprocessedTweets = this.parseTwitterFile(await fsp.readFile(outputPath + '/data/tweets.js', 'utf-8')); + + const tweets = unprocessedTweets.map(e => e.tweet); + const processedTweets = await this.recreateChain(['id_str'], ['in_reply_to_status_id_str'], tweets, false); + this.queueService.createImportTweetsToDbJob(job.data.user, processedTweets, null); + } finally { + cleanup(); + } + } else if (type === 'Facebook' || file.name.startsWith('facebook-') && file.name.endsWith('.zip')) { + const [path, cleanup] = await createTempDir(); + + this.logger.info(`Temp dir is ${path}`); + + const destPath = path + '/facebook.zip'; + + try { + await fsp.writeFile(destPath, '', 'binary'); + await this.downloadUrl(file.url, destPath); + } catch (e) { // TODO: 何度か再試行 + if (e instanceof Error || typeof e === 'string') { + this.logger.error(e); + } + throw e; + } + + const outputPath = path + '/facebook'; + try { + this.logger.succ(`Unzipping to ${outputPath}`); + ZipReader.withDestinationPath(outputPath).viaBuffer(await fsp.readFile(destPath)); + const postsJson = await fsp.readFile(outputPath + '/your_activity_across_facebook/posts/your_posts__check_ins__photos_and_videos_1.json', 'utf-8'); + const posts = JSON.parse(postsJson); + const facebookFolder = await this.driveFoldersRepository.findOneBy({ name: 'Facebook', userId: job.data.user.id, parentId: folder?.id }); + if (facebookFolder == null && folder) { + await this.driveFoldersRepository.insert({ id: this.idService.gen(), name: 'Facebook', userId: job.data.user.id, parentId: folder.id }); + const createdFolder = await this.driveFoldersRepository.findOneBy({ name: 'Facebook', userId: job.data.user.id, parentId: folder.id }); + if (createdFolder) await this.uploadFiles(outputPath + '/your_activity_across_facebook/posts/media', user, createdFolder.id); + } + this.queueService.createImportFBToDbJob(job.data.user, posts); + } finally { + cleanup(); + } + } else if (file.name.endsWith('.zip')) { + const [path, cleanup] = await createTempDir(); + + this.logger.info(`Temp dir is ${path}`); + + const destPath = path + '/unknown.zip'; + + try { + await fsp.writeFile(destPath, '', 'binary'); + await this.downloadUrl(file.url, destPath); + } catch (e) { // TODO: 何度か再試行 + if (e instanceof Error || typeof e === 'string') { + this.logger.error(e); + } + throw e; + } + + const outputPath = path + '/unknown'; + try { + this.logger.succ(`Unzipping to ${outputPath}`); + ZipReader.withDestinationPath(outputPath).viaBuffer(await fsp.readFile(destPath)); + const isInstagram = type === 'Instagram' || fs.existsSync(outputPath + '/instagram_live') || fs.existsSync(outputPath + '/instagram_ads_and_businesses'); + const isOutbox = type === 'Mastodon' || fs.existsSync(outputPath + '/outbox.json'); + if (isInstagram) { + const postsJson = await fsp.readFile(outputPath + '/content/posts_1.json', 'utf-8'); + const posts = JSON.parse(postsJson); + const igFolder = await this.driveFoldersRepository.findOneBy({ name: 'Instagram', userId: job.data.user.id, parentId: folder?.id }); + if (igFolder == null && folder) { + await this.driveFoldersRepository.insert({ id: this.idService.gen(), name: 'Instagram', userId: job.data.user.id, parentId: folder.id }); + const createdFolder = await this.driveFoldersRepository.findOneBy({ name: 'Instagram', userId: job.data.user.id, parentId: folder.id }); + if (createdFolder) await this.uploadFiles(outputPath + '/media/posts', user, createdFolder.id); + } + this.queueService.createImportIGToDbJob(job.data.user, posts); + } else if (isOutbox) { + const actorJson = await fsp.readFile(outputPath + '/actor.json', 'utf-8'); + const actor = JSON.parse(actorJson); + const isPleroma = actor['@context'].some((v: any) => typeof v === 'string' && v.match(/litepub(.*)/)); + if (isPleroma) { + const outboxJson = await fsp.readFile(outputPath + '/outbox.json', 'utf-8'); + const outbox = JSON.parse(outboxJson); + const processedToots = await this.recreateChain(['object', 'id'], ['object', 'inReplyTo'], outbox.orderedItems.filter((x: any) => x.type === 'Create' && x.object.type === 'Note'), true); + this.queueService.createImportPleroToDbJob(job.data.user, processedToots, null); + } else { + const outboxJson = await fsp.readFile(outputPath + '/outbox.json', 'utf-8'); + const outbox = JSON.parse(outboxJson); + let mastoFolder = await this.driveFoldersRepository.findOneBy({ name: 'Mastodon', userId: job.data.user.id, parentId: folder?.id }); + if (mastoFolder == null && folder) { + await this.driveFoldersRepository.insert({ id: this.idService.gen(), name: 'Mastodon', userId: job.data.user.id, parentId: folder.id }); + mastoFolder = await this.driveFoldersRepository.findOneBy({ name: 'Mastodon', userId: job.data.user.id, parentId: folder.id }); + } + if (fs.existsSync(outputPath + '/media_attachments/files') && mastoFolder) { + await this.uploadFiles(outputPath + '/media_attachments/files', user, mastoFolder.id); + } + const processedToots = await this.recreateChain(['object', 'id'], ['object', 'inReplyTo'], outbox.orderedItems.filter((x: any) => x.type === 'Create' && x.object.type === 'Note'), true); + this.queueService.createImportMastoToDbJob(job.data.user, processedToots, null); + } + } + } finally { + cleanup(); + } + } else if (job.data.type === 'Misskey' || file.name.startsWith('notes-') && file.name.endsWith('.json')) { + const [path, cleanup] = await createTemp(); + + this.logger.info(`Temp dir is ${path}`); + + try { + await fsp.writeFile(path, '', 'utf-8'); + await this.downloadUrl(file.url, path); + } catch (e) { // TODO: 何度か再試行 + if (e instanceof Error || typeof e === 'string') { + this.logger.error(e); + } + throw e; + } + + const notesJson = await fsp.readFile(path, 'utf-8'); + const notes = JSON.parse(notesJson); + const processedNotes = await this.recreateChain(['id'], ['replyId'], notes, false); + this.queueService.createImportKeyNotesToDbJob(job.data.user, processedNotes, null); + cleanup(); + } + + this.logger.succ('Import jobs created'); + } + + @bindThis + public async processKeyNotesToDb(job: Bull.Job): Promise { + const note = job.data.target; + const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); + if (user == null) { + return; + } + + if (note.renoteId) return; + + const parentNote = job.data.note ? await this.notesRepository.findOneBy({ id: job.data.note }) : null; + + const folder = await this.driveFoldersRepository.findOneBy({ name: 'Imports', userId: job.data.user.id }); + if (folder == null) return; + + const files: MiDriveFile[] = []; + const date = new Date(note.createdAt); + + if (note.files && this.isIterable(note.files)) { + let keyFolder = await this.driveFoldersRepository.findOneBy({ name: 'Misskey', userId: job.data.user.id, parentId: folder.id }); + if (keyFolder == null) { + await this.driveFoldersRepository.insert({ id: this.idService.gen(), name: 'Misskey', userId: job.data.user.id, parentId: folder.id }); + keyFolder = await this.driveFoldersRepository.findOneBy({ name: 'Misskey', userId: job.data.user.id, parentId: folder.id }); + } + + for await (const file of note.files) { + const [filePath, cleanup] = await createTemp(); + const slashdex = file.url.lastIndexOf('/'); + const name = file.url.substring(slashdex + 1); + + const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id }) ?? await this.driveFilesRepository.findOneBy({ name: name, userId: user.id, folderId: keyFolder?.id }); + + if (!exists) { + try { + await this.downloadUrl(file.url, filePath); + } catch (e) { // TODO: 何度か再試行 + this.logger.error(e instanceof Error ? e : new Error(e as string)); + } + const driveFile = await this.driveService.addFile({ + user: user, + path: filePath, + name: name, + folderId: keyFolder?.id, + }); + files.push(driveFile); + } else { + files.push(exists); + } + + cleanup(); + } + } + + const createdNote = await this.noteCreateService.import(user, { createdAt: date, reply: parentNote, text: note.text, apMentions: new Array(0), visibility: note.visibility, localOnly: note.localOnly, files: files, cw: note.cw }); + if (note.childNotes) this.queueService.createImportKeyNotesToDbJob(user, note.childNotes, createdNote.id); + } + + @bindThis + public async processMastoToDb(job: Bull.Job): Promise { + const toot = job.data.target; + const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); + if (user == null) { + return; + } + + const followers = toot.to.some((str: string) => str.includes('/followers')); + + if (toot.directMessage || !toot.to.includes('https://www.w3.org/ns/activitystreams#Public') && !followers) return; + + const visibility = followers ? toot.cc.includes('https://www.w3.org/ns/activitystreams#Public') ? 'home' : 'followers' : 'public'; + + const date = new Date(toot.object.published); + let text = undefined; + const files: MiDriveFile[] = []; + let reply: MiNote | null = null; + + if (toot.object.inReplyTo != null) { + const parentNote = job.data.note ? await this.notesRepository.findOneBy({ id: job.data.note }) : null; + if (parentNote) { + reply = parentNote; + } else { + try { + reply = await this.apNoteService.resolveNote(toot.object.inReplyTo); + } catch (error) { + reply = null; + } + } + } + + const hashtags = extractApHashtagObjects(toot.object.tag).map((x) => x.name).filter((x): x is string => x != null); + + try { + text = await this.mfmService.fromHtml(toot.object.content, hashtags); + } catch (error) { + text = undefined; + } + + if (toot.object.attachment && this.isIterable(toot.object.attachment)) { + for await (const file of toot.object.attachment) { + const slashdex = file.url.lastIndexOf('/'); + const name = file.url.substring(slashdex + 1); + const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id }); + if (exists) { + if (file.name) { + this.driveService.updateFile(exists, { comment: file.name }, user); + } + + files.push(exists); + } + } + } + + const createdNote = await this.noteCreateService.import(user, { createdAt: date, text: text, files: files, visibility: visibility, apMentions: new Array(0), cw: toot.object.sensitive ? toot.object.summary : null, reply: reply }); + if (toot.childNotes) this.queueService.createImportMastoToDbJob(user, toot.childNotes, createdNote.id); + } + + @bindThis + public async processPleroToDb(job: Bull.Job): Promise { + const post = job.data.target; + const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); + if (user == null) { + return; + } + + if (post.directMessage) return; + + const date = new Date(post.object.published); + let text = undefined; + const files: MiDriveFile[] = []; + let reply: MiNote | null = null; + + const folder = await this.driveFoldersRepository.findOneBy({ name: 'Imports', userId: job.data.user.id }); + if (folder == null) return; + + if (post.object.inReplyTo != null) { + const parentNote = job.data.note ? await this.notesRepository.findOneBy({ id: job.data.note }) : null; + if (parentNote) { + reply = parentNote; + } else { + try { + reply = await this.apNoteService.resolveNote(post.object.inReplyTo); + } catch (error) { + reply = null; + } + } + } + + const hashtags = extractApHashtagObjects(post.object.tag).map((x) => x.name).filter((x): x is string => x != null); + + try { + text = await this.mfmService.fromHtml(post.object.content, hashtags); + } catch (error) { + text = undefined; + } + + if (post.object.attachment && this.isIterable(post.object.attachment)) { + let pleroFolder = await this.driveFoldersRepository.findOneBy({ name: 'Pleroma', userId: job.data.user.id, parentId: folder.id }); + if (pleroFolder == null) { + await this.driveFoldersRepository.insert({ id: this.idService.gen(), name: 'Pleroma', userId: job.data.user.id, parentId: folder.id }); + pleroFolder = await this.driveFoldersRepository.findOneBy({ name: 'Pleroma', userId: job.data.user.id, parentId: folder.id }); + } + + for await (const file of post.object.attachment) { + const slashdex = file.url.lastIndexOf('/'); + const filename = file.url.substring(slashdex + 1); + const hash = crypto.createHash('md5').update(file.url).digest('base64url'); + const name = `${hash}-${filename}`; + const [filePath, cleanup] = await createTemp(); + + const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id }) ?? await this.driveFilesRepository.findOneBy({ name: name, userId: user.id, folderId: pleroFolder?.id }); + + if (!exists) { + try { + await this.downloadUrl(file.url, filePath); + } catch (e) { // TODO: 何度か再試行 + this.logger.error(e instanceof Error ? e : new Error(e as string)); + } + const driveFile = await this.driveService.addFile({ + user: user, + path: filePath, + name: name, + comment: file.name, + folderId: pleroFolder?.id, + }); + files.push(driveFile); + } else { + files.push(exists); + } + + cleanup(); + } + } + + const createdNote = await this.noteCreateService.import(user, { createdAt: date, text: text, files: files, apMentions: new Array(0), cw: post.object.sensitive ? post.object.summary : null, reply: reply }); + if (post.childNotes) this.queueService.createImportPleroToDbJob(user, post.childNotes, createdNote.id); + } + + @bindThis + public async processIGDb(job: Bull.Job): Promise { + const post = job.data.target; + const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); + if (user == null) { + return; + } + + let date; + let title; + const files: MiDriveFile[] = []; + + function decodeIGString(str: string) { + const arr = []; + for (let i = 0; i < str.length; i++) { + arr.push(str.charCodeAt(i)); + } + return Buffer.from(arr).toString('utf8'); + } + + if (post.media && this.isIterable(post.media) && post.media.length > 1) { + date = new Date(post.creation_timestamp * 1000); + title = decodeIGString(post.title); + for await (const file of post.media) { + const slashdex = file.uri.lastIndexOf('/'); + const name = file.uri.substring(slashdex + 1); + const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id }) ?? await this.driveFilesRepository.findOneBy({ name: `${name}.jpg`, userId: user.id }) ?? await this.driveFilesRepository.findOneBy({ name: `${name}.mp4`, userId: user.id }); + if (exists) { + files.push(exists); + } + } + } else if (post.media && this.isIterable(post.media) && !(post.media.length > 1)) { + date = new Date(post.media[0].creation_timestamp * 1000); + title = decodeIGString(post.media[0].title); + const slashdex = post.media[0].uri.lastIndexOf('/'); + const name = post.media[0].uri.substring(slashdex + 1); + const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id }) ?? await this.driveFilesRepository.findOneBy({ name: `${name}.jpg`, userId: user.id }) ?? await this.driveFilesRepository.findOneBy({ name: `${name}.mp4`, userId: user.id }); + if (exists) { + files.push(exists); + } + } + + await this.noteCreateService.import(user, { createdAt: date, text: title, files: files }); + } + + @bindThis + public async processTwitterDb(job: Bull.Job): Promise { + const tweet = job.data.target; + const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); + if (user == null) { + return; + } + + const folder = await this.driveFoldersRepository.findOneBy({ name: 'Imports', userId: job.data.user.id }); + if (folder == null) return; + + const parentNote = job.data.note ? await this.notesRepository.findOneBy({ id: job.data.note }) : null; + + async function replaceTwitterUrls(full_text: string, urls: any) { + let full_textedit = full_text; + urls.forEach((url: any) => { + full_textedit = full_textedit.replaceAll(url.url, url.expanded_url); + }); + return full_textedit; + } + + async function replaceTwitterMentions(full_text: string, mentions: any) { + let full_textedit = full_text; + mentions.forEach((mention: any) => { + full_textedit = full_textedit.replaceAll(`@${mention.screen_name}`, `[@${mention.screen_name}](https://twitter.com/${mention.screen_name})`); + }); + return full_textedit; + } + + try { + const date = new Date(tweet.created_at); + const decodedText = tweet.full_text.replaceAll('>', '>').replaceAll('<', '<').replaceAll('&', '&'); + const textReplaceURLs = tweet.entities.urls && tweet.entities.urls.length > 0 ? await replaceTwitterUrls(decodedText, tweet.entities.urls) : decodedText; + const text = tweet.entities.user_mentions && tweet.entities.user_mentions.length > 0 ? await replaceTwitterMentions(textReplaceURLs, tweet.entities.user_mentions) : textReplaceURLs; + const files: MiDriveFile[] = []; + + if (tweet.extended_entities && this.isIterable(tweet.extended_entities.media)) { + let twitFolder = await this.driveFoldersRepository.findOneBy({ name: 'Twitter', userId: job.data.user.id, parentId: folder.id }); + if (twitFolder == null) { + await this.driveFoldersRepository.insert({ id: this.idService.gen(), name: 'Twitter', userId: job.data.user.id, parentId: folder.id }); + twitFolder = await this.driveFoldersRepository.findOneBy({ name: 'Twitter', userId: job.data.user.id, parentId: folder.id }); + } + + for await (const file of tweet.extended_entities.media) { + if (file.video_info) { + const [filePath, cleanup] = await createTemp(); + const slashdex = file.video_info.variants[0].url.lastIndexOf('/'); + const name = file.video_info.variants[0].url.substring(slashdex + 1); + + const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id }) ?? await this.driveFilesRepository.findOneBy({ name: name, userId: user.id, folderId: twitFolder?.id }); + + const videos = file.video_info.variants.filter((x: any) => x.content_type === 'video/mp4'); + + if (!exists) { + try { + await this.downloadService.downloadUrl(videos[0].url, filePath); + } catch (e) { // TODO: 何度か再試行 + this.logger.error(e instanceof Error ? e : new Error(e as string)); + } + const driveFile = await this.driveService.addFile({ + user: user, + path: filePath, + name: name, + folderId: twitFolder?.id, + }); + files.push(driveFile); + } else { + files.push(exists); + } + + cleanup(); + } else if (file.media_url_https) { + const [filePath, cleanup] = await createTemp(); + const slashdex = file.media_url_https.lastIndexOf('/'); + const name = file.media_url_https.substring(slashdex + 1); + + const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id }); + + if (!exists) { + try { + await this.downloadService.downloadUrl(file.media_url_https, filePath); + } catch (e) { // TODO: 何度か再試行 + this.logger.error(e instanceof Error ? e : new Error(e as string)); + } + + const driveFile = await this.driveService.addFile({ + user: user, + path: filePath, + name: name, + folderId: twitFolder?.id, + }); + files.push(driveFile); + } else { + files.push(exists); + } + cleanup(); + } + } + } + const createdNote = await this.noteCreateService.import(user, { createdAt: date, reply: parentNote, text: text, files: files }); + if (tweet.childNotes) this.queueService.createImportTweetsToDbJob(user, tweet.childNotes, createdNote.id); + } catch (e) { + this.logger.warn(`Error: ${e}`); + } + } + + @bindThis + public async processFBDb(job: Bull.Job): Promise { + const post = job.data.target; + const user = await this.usersRepository.findOneBy({ id: job.data.user.id }); + if (user == null) { + return; + } + + if (!this.isIterable(post.data) || this.isIterable(post.data) && post.data[0].post === undefined) return; + + const date = new Date(post.timestamp * 1000); + const title = decodeFBString(post.data[0].post); + const files: MiDriveFile[] = []; + + function decodeFBString(str: string) { + const arr = []; + for (let i = 0; i < str.length; i++) { + arr.push(str.charCodeAt(i)); + } + return Buffer.from(arr).toString('utf8'); + } + + if (post.attachments && this.isIterable(post.attachments)) { + const media = []; + for await (const data of post.attachments[0].data) { + if (data.media) { + media.push(data.media); + } + } + + for await (const file of media) { + const slashdex = file.uri.lastIndexOf('/'); + const name = file.uri.substring(slashdex + 1); + const exists = await this.driveFilesRepository.findOneBy({ name: name, userId: user.id }); + if (exists) { + files.push(exists); + } + } + } + + await this.noteCreateService.import(user, { createdAt: date, text: title, files: files }); + } +} diff --git a/packages/backend/src/queue/types.ts b/packages/backend/src/queue/types.ts index b7d3c9b35e9b..e8bb6a0c404d 100644 --- a/packages/backend/src/queue/types.ts +++ b/packages/backend/src/queue/types.ts @@ -50,6 +50,13 @@ export type DbJobMap = { exportBlocking: DbJobDataWithUser; exportUserLists: DbJobDataWithUser; importAntennas: DBAntennaImportJobData; + importNotes: DbNoteImportJobData; + importTweetsToDb: DbNoteWithParentImportToDbJobData; + importIGToDb: DbNoteImportToDbJobData; + importFBToDb: DbNoteImportToDbJobData; + importMastoToDb: DbNoteWithParentImportToDbJobData; + importPleroToDb: DbNoteWithParentImportToDbJobData; + importKeyNotesToDb: DbNoteWithParentImportToDbJobData; importFollowing: DbUserImportJobData; importFollowingToDb: DbUserImportToDbJobData; importMuting: DbUserImportJobData; @@ -85,6 +92,12 @@ export type DbUserImportJobData = { withReplies?: boolean; }; +export type DbNoteImportJobData = { + user: ThinUser; + fileId: MiDriveFile['id']; + type?: string; +}; + export type DBAntennaImportJobData = { user: ThinUser, antenna: Antenna @@ -96,6 +109,17 @@ export type DbUserImportToDbJobData = { withReplies?: boolean; }; +export type DbNoteImportToDbJobData = { + user: ThinUser; + target: any; +}; + +export type DbNoteWithParentImportToDbJobData = { + user: ThinUser; + target: any; + note: MiNote['id'] | null; +}; + export type ObjectStorageJobData = ObjectStorageFileJobData | Record; export type ObjectStorageFileJobData = { diff --git a/packages/backend/src/server/NodeinfoServerService.ts b/packages/backend/src/server/NodeinfoServerService.ts index 3e0b6f27ae30..70830e54f17f 100644 --- a/packages/backend/src/server/NodeinfoServerService.ts +++ b/packages/backend/src/server/NodeinfoServerService.ts @@ -23,7 +23,7 @@ const nodeinfo_homepage = 'https://misskey-hub.net'; @Injectable() export class NodeinfoServerService { //semverに従って割り当てる - static reversiVersion = '1.0.0-yojo'; + static reversiVersion = '1.0.0-stream'; constructor( @Inject(DI.config) private config: Config, diff --git a/packages/backend/src/server/ServerModule.ts b/packages/backend/src/server/ServerModule.ts index 12d50619856c..2210ddb2b3b6 100644 --- a/packages/backend/src/server/ServerModule.ts +++ b/packages/backend/src/server/ServerModule.ts @@ -37,6 +37,7 @@ import { ChannelChannelService } from './api/stream/channels/channel.js'; import { DriveChannelService } from './api/stream/channels/drive.js'; import { GlobalTimelineChannelService } from './api/stream/channels/global-timeline.js'; import { HashtagChannelService } from './api/stream/channels/hashtag.js'; +import { HanamiTimelineChannelService } from './api/stream/channels/hanami-timeline.js'; import { HomeTimelineChannelService } from './api/stream/channels/home-timeline.js'; import { HybridTimelineChannelService } from './api/stream/channels/hybrid-timeline.js'; import { LocalTimelineChannelService } from './api/stream/channels/local-timeline.js'; @@ -84,6 +85,7 @@ import { ReversiGameChannelService } from './api/stream/channels/reversi-game.js RoleTimelineChannelService, ReversiChannelService, ReversiGameChannelService, + HanamiTimelineChannelService, HomeTimelineChannelService, HybridTimelineChannelService, LocalTimelineChannelService, diff --git a/packages/backend/src/server/api/EndpointsModule.ts b/packages/backend/src/server/api/EndpointsModule.ts index 41576bedaae7..c07b2f4970f7 100644 --- a/packages/backend/src/server/api/EndpointsModule.ts +++ b/packages/backend/src/server/api/EndpointsModule.ts @@ -228,6 +228,7 @@ import * as ep___i_gallery_likes from './endpoints/i/gallery/likes.js'; import * as ep___i_gallery_posts from './endpoints/i/gallery/posts.js'; import * as ep___i_importBlocking from './endpoints/i/import-blocking.js'; import * as ep___i_importFollowing from './endpoints/i/import-following.js'; +import * as ep___i_importNotes from './endpoints/i/import-notes.js'; import * as ep___i_importMuting from './endpoints/i/import-muting.js'; import * as ep___i_importUserLists from './endpoints/i/import-user-lists.js'; import * as ep___i_importAntennas from './endpoints/i/import-antennas.js'; @@ -285,6 +286,7 @@ import * as ep___notes_featured from './endpoints/notes/featured.js'; import * as ep___notes_globalTimeline from './endpoints/notes/global-timeline.js'; import * as ep___notes_hybridTimeline from './endpoints/notes/hybrid-timeline.js'; import * as ep___notes_localTimeline from './endpoints/notes/local-timeline.js'; +import * as ep___notes_hanamiTimeline from './endpoints/notes/hanami-timeline.js'; import * as ep___notes_mentions from './endpoints/notes/mentions.js'; import * as ep___notes_polls_recommendation from './endpoints/notes/polls/recommendation.js'; import * as ep___notes_polls_vote from './endpoints/notes/polls/vote.js'; @@ -611,6 +613,7 @@ const $i_gallery_likes: Provider = { provide: 'ep:i/gallery/likes', useClass: ep const $i_gallery_posts: Provider = { provide: 'ep:i/gallery/posts', useClass: ep___i_gallery_posts.default }; const $i_importBlocking: Provider = { provide: 'ep:i/import-blocking', useClass: ep___i_importBlocking.default }; const $i_importFollowing: Provider = { provide: 'ep:i/import-following', useClass: ep___i_importFollowing.default }; +const $i_importNotes: Provider = { provide: 'ep:i/import-notes', useClass: ep___i_importNotes.default }; const $i_importMuting: Provider = { provide: 'ep:i/import-muting', useClass: ep___i_importMuting.default }; const $i_importUserLists: Provider = { provide: 'ep:i/import-user-lists', useClass: ep___i_importUserLists.default }; const $i_importAntennas: Provider = { provide: 'ep:i/import-antennas', useClass: ep___i_importAntennas.default }; @@ -668,6 +671,7 @@ const $notes_featured: Provider = { provide: 'ep:notes/featured', useClass: ep__ const $notes_globalTimeline: Provider = { provide: 'ep:notes/global-timeline', useClass: ep___notes_globalTimeline.default }; const $notes_hybridTimeline: Provider = { provide: 'ep:notes/hybrid-timeline', useClass: ep___notes_hybridTimeline.default }; const $notes_localTimeline: Provider = { provide: 'ep:notes/local-timeline', useClass: ep___notes_localTimeline.default }; +const $notes_hanamiTimeline: Provider = { provide: 'ep:notes/hanami-timeline', useClass: ep___notes_hanamiTimeline.default }; const $notes_mentions: Provider = { provide: 'ep:notes/mentions', useClass: ep___notes_mentions.default }; const $notes_polls_recommendation: Provider = { provide: 'ep:notes/polls/recommendation', useClass: ep___notes_polls_recommendation.default }; const $notes_polls_vote: Provider = { provide: 'ep:notes/polls/vote', useClass: ep___notes_polls_vote.default }; @@ -998,6 +1002,7 @@ const $reversi_verify: Provider = { provide: 'ep:reversi/verify', useClass: ep__ $i_gallery_posts, $i_importBlocking, $i_importFollowing, + $i_importNotes, $i_importMuting, $i_importUserLists, $i_importAntennas, @@ -1055,6 +1060,7 @@ const $reversi_verify: Provider = { provide: 'ep:reversi/verify', useClass: ep__ $notes_globalTimeline, $notes_hybridTimeline, $notes_localTimeline, + $notes_hanamiTimeline, $notes_mentions, $notes_polls_recommendation, $notes_polls_vote, @@ -1379,6 +1385,7 @@ const $reversi_verify: Provider = { provide: 'ep:reversi/verify', useClass: ep__ $i_gallery_posts, $i_importBlocking, $i_importFollowing, + $i_importNotes, $i_importMuting, $i_importUserLists, $i_importAntennas, @@ -1436,6 +1443,7 @@ const $reversi_verify: Provider = { provide: 'ep:reversi/verify', useClass: ep__ $notes_globalTimeline, $notes_hybridTimeline, $notes_localTimeline, + $notes_hanamiTimeline, $notes_mentions, $notes_polls_recommendation, $notes_polls_vote, diff --git a/packages/backend/src/server/api/endpoints.ts b/packages/backend/src/server/api/endpoints.ts index 3dfb7fdad4c2..4a5143b2e56b 100644 --- a/packages/backend/src/server/api/endpoints.ts +++ b/packages/backend/src/server/api/endpoints.ts @@ -234,6 +234,7 @@ import * as ep___i_gallery_likes from './endpoints/i/gallery/likes.js'; import * as ep___i_gallery_posts from './endpoints/i/gallery/posts.js'; import * as ep___i_importBlocking from './endpoints/i/import-blocking.js'; import * as ep___i_importFollowing from './endpoints/i/import-following.js'; +import * as ep___i_importNotes from './endpoints/i/import-notes.js'; import * as ep___i_importMuting from './endpoints/i/import-muting.js'; import * as ep___i_importUserLists from './endpoints/i/import-user-lists.js'; import * as ep___i_importAntennas from './endpoints/i/import-antennas.js'; @@ -291,6 +292,7 @@ import * as ep___notes_featured from './endpoints/notes/featured.js'; import * as ep___notes_globalTimeline from './endpoints/notes/global-timeline.js'; import * as ep___notes_hybridTimeline from './endpoints/notes/hybrid-timeline.js'; import * as ep___notes_localTimeline from './endpoints/notes/local-timeline.js'; +import * as ep___notes_hanamiTimeline from './endpoints/notes/hanami-timeline.js'; import * as ep___notes_mentions from './endpoints/notes/mentions.js'; import * as ep___notes_polls_recommendation from './endpoints/notes/polls/recommendation.js'; import * as ep___notes_polls_vote from './endpoints/notes/polls/vote.js'; @@ -615,6 +617,7 @@ const eps = [ ['i/gallery/posts', ep___i_gallery_posts], ['i/import-blocking', ep___i_importBlocking], ['i/import-following', ep___i_importFollowing], + ['i/import-notes', ep___i_importNotes], ['i/import-muting', ep___i_importMuting], ['i/import-user-lists', ep___i_importUserLists], ['i/import-antennas', ep___i_importAntennas], @@ -672,6 +675,7 @@ const eps = [ ['notes/global-timeline', ep___notes_globalTimeline], ['notes/hybrid-timeline', ep___notes_hybridTimeline], ['notes/local-timeline', ep___notes_localTimeline], + ['notes/hanami-timeline', ep___notes_hanamiTimeline], ['notes/mentions', ep___notes_mentions], ['notes/polls/recommendation', ep___notes_polls_recommendation], ['notes/polls/vote', ep___notes_polls_vote], diff --git a/packages/backend/src/server/api/endpoints/i/import-notes.ts b/packages/backend/src/server/api/endpoints/i/import-notes.ts new file mode 100644 index 000000000000..4e0016355033 --- /dev/null +++ b/packages/backend/src/server/api/endpoints/i/import-notes.ts @@ -0,0 +1,72 @@ +import { Inject, Injectable } from '@nestjs/common'; +import ms from 'ms'; +import { Endpoint } from '@/server/api/endpoint-base.js'; +import { QueueService } from '@/core/QueueService.js'; +import type { DriveFilesRepository } from '@/models/_.js'; +import { DI } from '@/di-symbols.js'; +import { RoleService } from '@/core/RoleService.js'; +import { ApiError } from '../../error.js'; + +export const meta = { + secure: true, + requireCredential: true, + prohibitMoved: true, + limit: { + duration: ms('1hour'), + max: 2, + }, + + errors: { + noSuchFile: { + message: 'No such file.', + code: 'NO_SUCH_FILE', + id: 'b98644cf-a5ac-4277-a502-0b8054a709a3', + }, + + emptyFile: { + message: 'That file is empty.', + code: 'EMPTY_FILE', + id: '31a1b42c-06f7-42ae-8a38-a661c5c9f691', + }, + + notPermitted: { + message: 'You are not allowed to import notes.', + code: 'NO_PERMISSION', + id: '31a1b42c-06f7-42ae-8a38-a661c5c9f692', + }, + }, +} as const; + +export const paramDef = { + type: 'object', + properties: { + fileId: { type: 'string', format: 'misskey:id' }, + type: { type: 'string', nullable: true }, + }, + required: ['fileId'], +} as const; + +@Injectable() +export default class extends Endpoint { // eslint-disable-line import/no-default-export + constructor( + @Inject(DI.driveFilesRepository) + private driveFilesRepository: DriveFilesRepository, + + private queueService: QueueService, + private roleService: RoleService, + ) { + super(meta, paramDef, async (ps, me) => { + const file = await this.driveFilesRepository.findOneBy({ id: ps.fileId }); + + if (file == null) throw new ApiError(meta.errors.noSuchFile); + + if (file.size === 0) throw new ApiError(meta.errors.emptyFile); + + if ((await this.roleService.getUserPolicies(me.id)).canImportNotes === false) { + throw new ApiError(meta.errors.notPermitted); + } + + this.queueService.createImportNotesJob(me, file.id, ps.type); + }); + } +} diff --git a/packages/backend/src/server/api/endpoints/i/update.ts b/packages/backend/src/server/api/endpoints/i/update.ts index a1e2fa5e4cdf..01aa430d2389 100644 --- a/packages/backend/src/server/api/endpoints/i/update.ts +++ b/packages/backend/src/server/api/endpoints/i/update.ts @@ -173,6 +173,7 @@ export const paramDef = { preventAiLearning: { type: 'boolean' }, isBot: { type: 'boolean' }, isCat: { type: 'boolean' }, + isInHanaMode: { type: 'boolean' }, injectFeaturedNote: { type: 'boolean' }, receiveAnnouncementEmail: { type: 'boolean' }, alwaysMarkNsfw: { type: 'boolean' }, @@ -322,6 +323,7 @@ export default class extends Endpoint { // eslint- if (typeof ps.noCrawle === 'boolean') profileUpdates.noCrawle = ps.noCrawle; if (typeof ps.preventAiLearning === 'boolean') profileUpdates.preventAiLearning = ps.preventAiLearning; if (typeof ps.isCat === 'boolean') updates.isCat = ps.isCat; + if (typeof ps.isInHanaMode === 'boolean') updates.isInHanaMode = ps.isInHanaMode; if (typeof ps.injectFeaturedNote === 'boolean') profileUpdates.injectFeaturedNote = ps.injectFeaturedNote; if (typeof ps.receiveAnnouncementEmail === 'boolean') profileUpdates.receiveAnnouncementEmail = ps.receiveAnnouncementEmail; if (typeof ps.alwaysMarkNsfw === 'boolean') { diff --git a/packages/backend/src/server/api/endpoints/notes/featured.ts b/packages/backend/src/server/api/endpoints/notes/featured.ts index dcd971360d2e..62f8f97047c1 100644 --- a/packages/backend/src/server/api/endpoints/notes/featured.ts +++ b/packages/backend/src/server/api/endpoints/notes/featured.ts @@ -87,6 +87,7 @@ export default class extends Endpoint { // eslint- const query = this.notesRepository.createQueryBuilder('note') .where('note.id IN (:...noteIds)', { noteIds: noteIds }) + .andWhere('note.userHost IS NULL') .innerJoinAndSelect('note.user', 'user') .leftJoinAndSelect('note.reply', 'reply') .leftJoinAndSelect('note.renote', 'renote') diff --git a/packages/backend/src/server/api/endpoints/notes/hanami-timeline.ts b/packages/backend/src/server/api/endpoints/notes/hanami-timeline.ts new file mode 100644 index 000000000000..b943af7641ef --- /dev/null +++ b/packages/backend/src/server/api/endpoints/notes/hanami-timeline.ts @@ -0,0 +1,341 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ +// ホームタイムラインとfeaturedから持ってくる +// UntilIDが3日より前だった場合はもうfeauturedから取得しない（feauturedがそれより前のデータを持っていないため） +// ホームライムラインからの結果が最小のID（最古のノートになるように返す）ページネーションが壊れる+ホームタイムラインの結果に抜け漏れが発生するため +// feauturedに抜け漏れが出るのはTODO +// featuredのミュートとブロックを確認→2つの結果を比べる→feauturedをホームタイムラインの結果より新しくなるようにトリム→2つの結果を一意にしつつlimitでトリム→id順にソートしてreturn + +import { Brackets } from 'typeorm'; +import { Inject, Injectable } from '@nestjs/common'; +import type { NotesRepository, ChannelFollowingsRepository } from '@/models/_.js'; +import { Endpoint } from '@/server/api/endpoint-base.js'; +import { QueryService } from '@/core/QueryService.js'; +import ActiveUsersChart from '@/core/chart/charts/active-users.js'; +import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; +import { DI } from '@/di-symbols.js'; +import { RoleService } from '@/core/RoleService.js'; +import { IdService } from '@/core/IdService.js'; +import { CacheService } from '@/core/CacheService.js'; +import { UserFollowingService } from '@/core/UserFollowingService.js'; +import { MiLocalUser } from '@/models/User.js'; +import { MetaService } from '@/core/MetaService.js'; +import { FanoutTimelineEndpointService } from '@/core/FanoutTimelineEndpointService.js'; +import { FeaturedService } from '@/core/FeaturedService.js'; +import { isUserRelated } from '@/misc/is-user-related.js'; +import { ApiError } from '../../error.js'; + +export const meta = { + tags: ['notes'], + + requireCredential: true, + kind: 'read:account', + + res: { + type: 'array', + optional: false, nullable: false, + items: { + type: 'object', + optional: false, nullable: false, + ref: 'Note', + }, + }, + + errors: { + HanamiTlDisabled: { + message: 'Hanami timeline has been disabled.', + code: 'HanamiTL_DISABLED', + id: 'ffa57e0f-d14e-48d6-a64c-8fbcba5635ab', + }, + }, +} as const; + +export const paramDef = { + type: 'object', + properties: { + limit: { type: 'integer', minimum: 1, maximum: 100, default: 10 }, + sinceId: { type: 'string', format: 'misskey:id' }, + untilId: { type: 'string', format: 'misskey:id' }, + sinceDate: { type: 'integer' }, + untilDate: { type: 'integer' }, + allowPartial: { type: 'boolean', default: false }, // true is recommended but for compatibility false by default + includeMyRenotes: { type: 'boolean', default: true }, + includeRenotedMyNotes: { type: 'boolean', default: true }, + includeLocalRenotes: { type: 'boolean', default: true }, + withFiles: { type: 'boolean', default: false }, + withRenotes: { type: 'boolean', default: true }, + }, + required: [], +} as const; + +@Injectable() +export default class extends Endpoint { // eslint-disable-line import/no-default-export + private globalNotesRankingCache: string[] = []; + private globalNotesRankingCacheLastFetchedAt = 0; + + constructor( + @Inject(DI.notesRepository) + private notesRepository: NotesRepository, + + @Inject(DI.channelFollowingsRepository) + private channelFollowingsRepository: ChannelFollowingsRepository, + + private noteEntityService: NoteEntityService, + private roleService: RoleService, + private activeUsersChart: ActiveUsersChart, + private idService: IdService, + private cacheService: CacheService, + private fanoutTimelineEndpointService: FanoutTimelineEndpointService, + private userFollowingService: UserFollowingService, + private queryService: QueryService, + private metaService: MetaService, + private featuredService: FeaturedService, + ) { + super(meta, paramDef, async (ps, me) => { + const untilId = ps.untilId ?? (ps.untilDate ? this.idService.gen(ps.untilDate!) : null); + const sinceId = ps.sinceId ?? (ps.sinceDate ? this.idService.gen(ps.sinceDate!) : null); + + const policies = await this.roleService.getUserPolicies(me ? me.id : null); + if (!policies.hanamiTlAvailable) { + throw new ApiError(meta.errors.HanamiTlDisabled); + } + + const serverSettings = await this.metaService.fetch(); + + if (!serverSettings.enableFanoutTimeline) { + const timeline = await this.getFromDb({ + untilId, + sinceId, + limit: ps.limit, + includeMyRenotes: ps.includeMyRenotes, + includeRenotedMyNotes: ps.includeRenotedMyNotes, + includeLocalRenotes: ps.includeLocalRenotes, + withFiles: ps.withFiles, + withRenotes: ps.withRenotes, + }, me); + + process.nextTick(() => { + this.activeUsersChart.read(me); + }); + + return await this.noteEntityService.packMany(timeline, me); + } + + const [ + followings, + ] = await Promise.all([ + this.cacheService.userFollowingsCache.fetch(me.id), + ]); + + const packedHomeTimelineNotes = await this.fanoutTimelineEndpointService.timeline({ + untilId, + sinceId, + limit: ps.limit, + allowPartial: ps.allowPartial, + me, + useDbFallback: serverSettings.enableFanoutTimelineDbFallback, + redisTimelines: ps.withFiles ? [`homeTimelineWithFiles:${me.id}`] : [`homeTimeline:${me.id}`], + alwaysIncludeMyNotes: true, + excludePureRenotes: !ps.withRenotes, + noteFilter: note => { + if (note.reply && note.reply.visibility === 'followers') { + if (!Object.hasOwn(followings, note.reply.userId) && note.reply.userId !== me.id) return false; + } + + return true; + }, + dbFallback: async (untilId, sinceId, limit) => await this.getFromDb({ + untilId, + sinceId, + limit, + includeMyRenotes: ps.includeMyRenotes, + includeRenotedMyNotes: ps.includeRenotedMyNotes, + includeLocalRenotes: ps.includeLocalRenotes, + withFiles: ps.withFiles, + withRenotes: ps.withRenotes, + }, me), + }); + + process.nextTick(() => { + this.activeUsersChart.read(me); + }); + + // 3日経っていないことを確認 + if (ps.untilId) { + if (this.idService.parse(ps.untilId).date.getTime() < Date.now() - 1000 * 60 * 60 * 24 * 3 ) { + return packedHomeTimelineNotes; + } + } + + let feauturedNoteIds: string[]; + if (this.globalNotesRankingCacheLastFetchedAt !== 0 && (Date.now() - this.globalNotesRankingCacheLastFetchedAt < 1000 * 60 * 30)) { + feauturedNoteIds = this.globalNotesRankingCache; + } else { + feauturedNoteIds = await this.featuredService.getGlobalNotesRanking(100); + this.globalNotesRankingCache = feauturedNoteIds; + this.globalNotesRankingCacheLastFetchedAt = Date.now(); + } + + // feauturedのノート数が0でないことを確認 + if (feauturedNoteIds.length === 0) { + return packedHomeTimelineNotes; + } + + const [ + userIdsWhoMeMuting, + userIdsWhoBlockingMe, + ] = await Promise.all([ + this.cacheService.userMutingsCache.fetch(me.id), + this.cacheService.userBlockedCache.fetch(me.id), + ]); + + const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId) + .where('note.id IN (:...noteIds)', { noteIds: feauturedNoteIds }) + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser') + .leftJoinAndSelect('note.channel', 'channel'); + + const feauturedNotes = (await query.getMany()).filter(note => { + if (isUserRelated(note, userIdsWhoBlockingMe)) return false; + if (isUserRelated(note, userIdsWhoMeMuting)) return false; + + return true; + }); + + // 取得した中で最古のホームタイムラインのnoteIDを抽出する + const minHomeTimelineId = packedHomeTimelineNotes.length > 0 ? packedHomeTimelineNotes[packedHomeTimelineNotes.length - 1].id : null; + // 結果を一意にした上で最古のnoteIdがホームタイムライン由来にする + const filteredFeaturedNotes = feauturedNotes.filter(note => { + if (!minHomeTimelineId) return true; + return note.id < minHomeTimelineId; + }); + + if (filteredFeaturedNotes.length === 0) { + return packedHomeTimelineNotes; + } + + const packedFeauturedNotes = await this.noteEntityService.packMany(filteredFeaturedNotes, me); + + const allNotes = [...packedHomeTimelineNotes, ...packedFeauturedNotes] + .sort((a, b) => a.id > b.id ? -1 : 1) + .filter((note, index, self) => + index === self.findIndex(n => n.id === note.id), // 一意にする + ) + .slice(0, ps.limit); // ps.limitでトリム + + return allNotes; + }); + } + + private async getFromDb(ps: { untilId: string | null; sinceId: string | null; limit: number; includeMyRenotes: boolean; includeRenotedMyNotes: boolean; includeLocalRenotes: boolean; withFiles: boolean; withRenotes: boolean; }, me: MiLocalUser) { + const followees = await this.userFollowingService.getFollowees(me.id); + const followingChannels = await this.channelFollowingsRepository.find({ + where: { + followerId: me.id, + }, + }); + + //#region Construct query + const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId) + .innerJoinAndSelect('note.user', 'user') + .leftJoinAndSelect('note.reply', 'reply') + .leftJoinAndSelect('note.renote', 'renote') + .leftJoinAndSelect('reply.user', 'replyUser') + .leftJoinAndSelect('renote.user', 'renoteUser'); + + if (followees.length > 0 && followingChannels.length > 0) { + // ユーザー・チャンネルともにフォローあり + const meOrFolloweeIds = [me.id, ...followees.map(f => f.followeeId)]; + const followingChannelIds = followingChannels.map(x => x.followeeId); + query.andWhere(new Brackets(qb => { + qb + .where(new Brackets(qb2 => { + qb2 + .where('note.userId IN (:...meOrFolloweeIds)', { meOrFolloweeIds: meOrFolloweeIds }) + .andWhere('note.channelId IS NULL'); + })) + .orWhere('note.channelId IN (:...followingChannelIds)', { followingChannelIds }); + })); + } else if (followees.length > 0) { + // ユーザーフォローのみ（チャンネルフォローなし） + const meOrFolloweeIds = [me.id, ...followees.map(f => f.followeeId)]; + query + .andWhere('note.channelId IS NULL') + .andWhere('note.userId IN (:...meOrFolloweeIds)', { meOrFolloweeIds: meOrFolloweeIds }); + } else if (followingChannels.length > 0) { + // チャンネルフォローのみ（ユーザーフォローなし） + const followingChannelIds = followingChannels.map(x => x.followeeId); + query.andWhere(new Brackets(qb => { + qb + .where('note.channelId IN (:...followingChannelIds)', { followingChannelIds }) + .orWhere('note.userId = :meId', { meId: me.id }); + })); + } else { + // フォローなし + query + .andWhere('note.channelId IS NULL') + .andWhere('note.userId = :meId', { meId: me.id }); + } + + query.andWhere(new Brackets(qb => { + qb + .where('note.replyId IS NULL') // 返信ではない + .orWhere(new Brackets(qb => { + qb // 返信だけど投稿者自身への返信 + .where('note.replyId IS NOT NULL') + .andWhere('note.replyUserId = note.userId'); + })); + })); + + this.queryService.generateVisibilityQuery(query, me); + this.queryService.generateMutedUserQuery(query, me); + this.queryService.generateBlockedUserQuery(query, me); + this.queryService.generateMutedUserRenotesQueryForNotes(query, me); + + if (ps.includeMyRenotes === false) { + query.andWhere(new Brackets(qb => { + qb.orWhere('note.userId != :meId', { meId: me.id }); + qb.orWhere('note.renoteId IS NULL'); + qb.orWhere('note.text IS NOT NULL'); + qb.orWhere('note.fileIds != \'{}\''); + qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)'); + })); + } + + if (ps.includeRenotedMyNotes === false) { + query.andWhere(new Brackets(qb => { + qb.orWhere('note.renoteUserId != :meId', { meId: me.id }); + qb.orWhere('note.renoteId IS NULL'); + qb.orWhere('note.text IS NOT NULL'); + qb.orWhere('note.fileIds != \'{}\''); + qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)'); + })); + } + + if (ps.includeLocalRenotes === false) { + query.andWhere(new Brackets(qb => { + qb.orWhere('note.renoteUserHost IS NOT NULL'); + qb.orWhere('note.renoteId IS NULL'); + qb.orWhere('note.text IS NOT NULL'); + qb.orWhere('note.fileIds != \'{}\''); + qb.orWhere('0 < (SELECT COUNT(*) FROM poll WHERE poll."noteId" = note.id)'); + })); + } + + if (ps.withFiles) { + query.andWhere('note.fileIds != \'{}\''); + } + + if (ps.withRenotes === false) { + query.andWhere('note.renoteId IS NULL'); + } + //#endregion + + return await query.limit(ps.limit).getMany(); + } +} diff --git a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts index 2a2c6599427d..2756775f730c 100644 --- a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts @@ -47,7 +47,7 @@ export const meta = { bothWithRepliesAndWithFiles: { message: 'Specifying both withReplies and withFiles is not supported', code: 'BOTH_WITH_REPLIES_AND_WITH_FILES', - id: 'dfaa3eb7-8002-4cb7-bcc4-1095df46656f' + id: 'dfaa3eb7-8002-4cb7-bcc4-1095df46656f', }, }, } as const; @@ -208,10 +208,10 @@ export default class extends Endpoint { // eslint- if (followees.length > 0) { const meOrFolloweeIds = [me.id, ...followees.map(f => f.followeeId)]; qb.where('note.userId IN (:...meOrFolloweeIds)', { meOrFolloweeIds: meOrFolloweeIds }); - qb.orWhere('(note.visibility = \'public\') AND (note.userHost IS NULL)'); + qb.orWhere('(note.visibility = \'public\') AND (note.userHost IS NULL) AND (note.isNoteInHanaMode IS FALSE)'); } else { qb.where('note.userId = :meId', { meId: me.id }); - qb.orWhere('(note.visibility = \'public\') AND (note.userHost IS NULL)'); + qb.orWhere('(note.visibility = \'public\') AND (note.userHost IS NULL) AND (note.isNoteInHanaMode IS FALSE)'); } })) .innerJoinAndSelect('note.user', 'user') diff --git a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts index be82b5a8a749..746a9eb2df38 100644 --- a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts @@ -151,7 +151,7 @@ export default class extends Endpoint { // eslint- }, me: MiLocalUser | null) { const query = this.queryService.makePaginationQuery(this.notesRepository.createQueryBuilder('note'), ps.sinceId, ps.untilId) - .andWhere('(note.visibility = \'public\') AND (note.userHost IS NULL) AND (note.channelId IS NULL)') + .andWhere('(note.visibility = \'public\') AND (note.userHost IS NULL) AND (note.channelId IS NULL) AND (note.isNoteInHanaMode IS FALSE)') .innerJoinAndSelect('note.user', 'user') .leftJoinAndSelect('note.reply', 'reply') .leftJoinAndSelect('note.renote', 'renote') diff --git a/packages/backend/src/server/api/stream/ChannelsService.ts b/packages/backend/src/server/api/stream/ChannelsService.ts index 253409259fad..3d36908b2198 100644 --- a/packages/backend/src/server/api/stream/ChannelsService.ts +++ b/packages/backend/src/server/api/stream/ChannelsService.ts @@ -8,6 +8,7 @@ import { bindThis } from '@/decorators.js'; import { HybridTimelineChannelService } from './channels/hybrid-timeline.js'; import { LocalTimelineChannelService } from './channels/local-timeline.js'; import { HomeTimelineChannelService } from './channels/home-timeline.js'; +import { HanamiTimelineChannelService } from './channels/hanami-timeline.js'; import { GlobalTimelineChannelService } from './channels/global-timeline.js'; import { MainChannelService } from './channels/main.js'; import { ChannelChannelService } from './channels/channel.js'; @@ -27,6 +28,7 @@ import { type MiChannelService } from './channel.js'; export class ChannelsService { constructor( private mainChannelService: MainChannelService, + private hanamiTimelineChannelService: HanamiTimelineChannelService, private homeTimelineChannelService: HomeTimelineChannelService, private localTimelineChannelService: LocalTimelineChannelService, private hybridTimelineChannelService: HybridTimelineChannelService, @@ -49,6 +51,7 @@ export class ChannelsService { public getChannelService(name: string): MiChannelService { switch (name) { case 'main': return this.mainChannelService; + case 'hanamiTimeline': return this.hanamiTimelineChannelService; case 'homeTimeline': return this.homeTimelineChannelService; case 'localTimeline': return this.localTimelineChannelService; case 'hybridTimeline': return this.hybridTimelineChannelService; diff --git a/packages/backend/src/server/api/stream/channels/hanami-timeline.ts b/packages/backend/src/server/api/stream/channels/hanami-timeline.ts new file mode 100644 index 000000000000..367affa3d3a8 --- /dev/null +++ b/packages/backend/src/server/api/stream/channels/hanami-timeline.ts @@ -0,0 +1,127 @@ +/* + * SPDX-FileCopyrightText: syuilo and misskey-project + * SPDX-License-Identifier: AGPL-3.0-only + */ + +import { Injectable } from '@nestjs/common'; +import type { Packed } from '@/misc/json-schema.js'; +import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; +import { bindThis } from '@/decorators.js'; +import { RoleService } from '@/core/RoleService.js'; +import { isRenotePacked, isQuotePacked } from '@/misc/is-renote.js'; +import type { JsonObject } from '@/misc/json-value.js'; +import Channel, { type MiChannelService } from '../channel.js'; + +class HanamiTimelineChannel extends Channel { + public readonly chName = 'hanamiTimeline'; + public static shouldShare = false; + public static requireCredential = true as const; + public static kind = 'read:account'; + private withRenotes: boolean; + private withFiles: boolean; + + constructor( + private noteEntityService: NoteEntityService, + private roleService: RoleService, + + id: string, + connection: Channel['connection'], + ) { + super(id, connection); + //this.onNote = this.onNote.bind(this); + } + + @bindThis + public async init(params: JsonObject): Promise { + const policies = await this.roleService.getUserPolicies(this.user ? this.user.id : null); + if (!policies.hanamiTlAvailable) return; + + this.withRenotes = !!(params.withRenotes ?? true); + this.withFiles = !!(params.withFiles ?? false); + + this.subscriber.on('notesStream', this.onNote); + } + + @bindThis + private async onNote(note: Packed<'Note'>) { + const isMe = this.user!.id === note.userId; + + if (this.withFiles && (note.fileIds == null || note.fileIds.length === 0)) return; + + if (note.channelId) { + if (!this.followingChannels.has(note.channelId)) return; + } else { + // その投稿のユーザーをフォローしていなかったら弾く + if (!isMe && !Object.hasOwn(this.following, note.userId)) return; + } + + if (note.visibility === 'followers') { + if (!isMe && !Object.hasOwn(this.following, note.userId)) return; + } else if (note.visibility === 'specified') { + if (!isMe && !note.visibleUserIds!.includes(this.user!.id)) return; + } + + if (note.reply) { + const reply = note.reply; + if (this.following[note.userId]?.withReplies) { + // 自分のフォローしていないユーザーの visibility: followers な投稿への返信は弾く + if (reply.visibility === 'followers' && !Object.hasOwn(this.following, reply.userId) && reply.userId !== this.user!.id) return; + } else { + // 「チャンネル接続主への返信」でもなければ、「チャンネル接続主が行った返信」でもなければ、「投稿者の投稿者自身への返信」でもない場合 + if (reply.userId !== this.user!.id && !isMe && reply.userId !== note.userId) return; + } + } + + // 純粋なリノート（引用リノートでないリノート）の場合 + if (isRenotePacked(note) && !isQuotePacked(note) && note.renote) { + if (!this.withRenotes) return; + if (note.renote.reply) { + const reply = note.renote.reply; + // 自分のフォローしていないユーザーの visibility: followers な投稿への返信のリノートは弾く + if (reply.visibility === 'followers' && !Object.hasOwn(this.following, reply.userId) && reply.userId !== this.user!.id) return; + } + } + + if (this.isNoteMutedOrBlocked(note)) return; + + if (this.user && isRenotePacked(note) && !isQuotePacked(note)) { + if (note.renote && Object.keys(note.renote.reactions).length > 0) { + const myRenoteReaction = await this.noteEntityService.populateMyReaction(note.renote, this.user.id); + note.renote.myReaction = myRenoteReaction; + } + } + + this.connection.cacheNote(note); + + this.send('note', note); + } + + @bindThis + public dispose() { + // Unsubscribe events + this.subscriber.off('notesStream', this.onNote); + } +} + +@Injectable() +export class HanamiTimelineChannelService implements MiChannelService { + public readonly shouldShare = HanamiTimelineChannel.shouldShare; + public readonly requireCredential = HanamiTimelineChannel.requireCredential; + public readonly kind = HanamiTimelineChannel.kind; + + constructor( + private noteEntityService: NoteEntityService, + private roleService: RoleService, + ) { + } + + @bindThis + public create(id: string, connection: Channel['connection']): HanamiTimelineChannel { + return new HanamiTimelineChannel( + this.noteEntityService, + this.roleService, + id, + connection, + ); + } +} diff --git a/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts b/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts index 75bd13221f10..c82a8d5a7039 100644 --- a/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts @@ -70,6 +70,11 @@ class HybridTimelineChannel extends Channel { if (!isMe && !note.visibleUserIds!.includes(this.user!.id)) return; } + // はなモードが有効な投稿はフォローしている人だけ配信 + if (note.isNoteInHanaMode) { + if (!isMe && !Object.hasOwn(this.following, note.userId)) return; + }; + if (this.isNoteMutedOrBlocked(note)) return; if (note.reply) { diff --git a/packages/backend/src/server/api/stream/channels/local-timeline.ts b/packages/backend/src/server/api/stream/channels/local-timeline.ts index 491029f5dea1..f52171ee7864 100644 --- a/packages/backend/src/server/api/stream/channels/local-timeline.ts +++ b/packages/backend/src/server/api/stream/channels/local-timeline.ts @@ -53,6 +53,7 @@ class LocalTimelineChannel extends Channel { if (note.user.host !== null) return; if (note.visibility !== 'public') return; if (note.channelId != null) return; + if (note.isNoteInHanaMode) return; // 関係ない返信は除外 if (note.reply && this.user && !this.following[note.userId]?.withReplies && !this.withReplies) { diff --git a/packages/backend/src/server/web/views/base.pug b/packages/backend/src/server/web/views/base.pug index ff8970246819..2f31e087edbd 100644 --- a/packages/backend/src/server/web/views/base.pug +++ b/packages/backend/src/server/web/views/base.pug @@ -37,6 +37,7 @@ html link(rel='prefetch' href=notFoundImageUrl) //- https://github.com/misskey-dev/misskey/issues/9842 link(rel='stylesheet' href='/assets/tabler-icons/tabler-icons.min.css?v3.3.0') + link(rel='stylesheet' href='https://static-assets.misskey.flowers/fonts/hana-icons/font.css') link(rel='modulepreload' href=`/vite/${clientEntry.file}`) if !config.clientManifestExists diff --git a/packages/backend/test/e2e/users.ts b/packages/backend/test/e2e/users.ts index 61fd7599322a..d84e0832e3ca 100644 --- a/packages/backend/test/e2e/users.ts +++ b/packages/backend/test/e2e/users.ts @@ -40,6 +40,7 @@ describe('ユーザー', () => { avatarDecorations: user.avatarDecorations, isBot: user.isBot, isCat: user.isCat, + isInHanaMode: user.isInHanaMode, instance: user.instance, emojis: user.emojis, onlineStatus: user.onlineStatus, @@ -442,6 +443,8 @@ describe('ユーザー', () => { { parameters: () => ({ isBot: false }) }, { parameters: () => ({ isCat: true }) }, { parameters: () => ({ isCat: false }) }, + { parameters: () => ({ isInHanaMode: true }) }, + { parameters: () => ({ isInHanaMode: false }) }, { parameters: () => ({ injectFeaturedNote: true }) }, { parameters: () => ({ injectFeaturedNote: false }) }, { parameters: () => ({ receiveAnnouncementEmail: true }) }, diff --git a/packages/backend/test/unit/NoteCreateService.ts b/packages/backend/test/unit/NoteCreateService.ts index f2d4c8ffbb77..87873c334828 100644 --- a/packages/backend/test/unit/NoteCreateService.ts +++ b/packages/backend/test/unit/NoteCreateService.ts @@ -36,6 +36,7 @@ describe('NoteCreateService', () => { userId: 'some-user-id', user: null, localOnly: false, + isNoteInHanaMode: false, reactionAcceptance: null, renoteCount: 0, repliesCount: 0, diff --git a/packages/backend/test/unit/misc/is-renote.ts b/packages/backend/test/unit/misc/is-renote.ts index 0b713e8bf6b4..b2fd863a1724 100644 --- a/packages/backend/test/unit/misc/is-renote.ts +++ b/packages/backend/test/unit/misc/is-renote.ts @@ -19,6 +19,7 @@ const base: MiNote = { userId: 'some-user-id', user: null, localOnly: false, + isNoteInHanaMode: false, reactionAcceptance: null, renoteCount: 0, repliesCount: 0, diff --git a/packages/frontend/src/components/HanaHanaModeSwitcher.vue b/packages/frontend/src/components/HanaHanaModeSwitcher.vue new file mode 100644 index 000000000000..541ad167d7f7 --- /dev/null +++ b/packages/frontend/src/components/HanaHanaModeSwitcher.vue @@ -0,0 +1,167 @@ + + + + + diff --git a/packages/frontend/src/components/MkNote.vue b/packages/frontend/src/components/MkNote.vue index 917de150693d..f3135336b7a7 100644 --- a/packages/frontend/src/components/MkNote.vue +++ b/packages/frontend/src/components/MkNote.vue @@ -31,6 +31,7 @@ SPDX-License-Identifier: AGPL-3.0-only + diff --git a/packages/frontend/src/components/MkNoteDetailed.vue b/packages/frontend/src/components/MkNoteDetailed.vue index f6cc5e5486f7..4bf8b1ec9fb2 100644 --- a/packages/frontend/src/components/MkNoteDetailed.vue +++ b/packages/frontend/src/components/MkNoteDetailed.vue @@ -36,6 +36,7 @@ SPDX-License-Identifier: AGPL-3.0-only + @@ -55,6 +56,7 @@ SPDX-License-Identifier: AGPL-3.0-only bot
+ diff --git a/packages/frontend/src/components/MkNoteHeader.vue b/packages/frontend/src/components/MkNoteHeader.vue index 0a08a8cd4391..594130e18a62 100644 --- a/packages/frontend/src/components/MkNoteHeader.vue +++ b/packages/frontend/src/components/MkNoteHeader.vue @@ -23,6 +23,7 @@ SPDX-License-Identifier: AGPL-3.0-only + diff --git a/packages/frontend/src/components/MkPostForm.vue b/packages/frontend/src/components/MkPostForm.vue index dd12ec3d42f7..4a67cd7bdaa9 100644 --- a/packages/frontend/src/components/MkPostForm.vue +++ b/packages/frontend/src/components/MkPostForm.vue @@ -21,11 +21,14 @@ SPDX-License-Identifier: AGPL-3.0-only