Merge branch 'main' of https://github.com/tangly1024/NotionNext

.env.local

@@ -1,5 +1,5 @@
 # 环境变量 @see https://www.nextjs.cn/docs/basic-features/environment-variables
-NEXT_PUBLIC_VERSION=4.5.2
+NEXT_PUBLIC_VERSION=4.5.3
 
 
 # 可在此添加环境变量，去掉最左边的（# ）注释即可

components/ExternalPlugins.js

@@ -101,7 +101,7 @@ const ExternalPlugin = props => {
     // 异步渲染谷歌广告
     if (ADSENSE_GOOGLE_ID) {
       setTimeout(() => {
-        initGoogleAdsense()
+        initGoogleAdsense(ADSENSE_GOOGLE_ID)
       }, 1000)
     }
 

components/GoogleAdsense.js

@@ -61,10 +61,10 @@ function getNodesWithAdsByGoogleClass(node) {
  * 初始化谷歌广告
  * @returns
  */
-export const initGoogleAdsense = async () => {
+export const initGoogleAdsense = async ADSENSE_GOOGLE_ID => {
   console.log('Load Adsense')
   loadExternalResource(
-    `https://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js?client=${siteConfig('ADSENSE_GOOGLE_ID')}`,
+    `https://pagead2.googlesyndication.com/pagead/js/adsbygoogle.js?client=${ADSENSE_GOOGLE_ID}`,
     'js'
   ).then(url => {
     setTimeout(() => {
@@ -109,7 +109,9 @@ export const initGoogleAdsense = async () => {
  * 添加 可以在本地调试
  */
 const AdSlot = ({ type = 'show' }) => {
-  if (!siteConfig('ADSENSE_GOOGLE_ID')) {
+  const ADSENSE_GOOGLE_ID = siteConfig('ADSENSE_GOOGLE_ID')
+  const ADSENSE_GOOGLE_TEST = siteConfig('ADSENSE_GOOGLE_TEST')
+  if (!ADSENSE_GOOGLE_ID) {
     return null
   }
   // 文章内嵌广告
@@ -120,8 +122,8 @@ const AdSlot = ({ type = 'show' }) => {
         style={{ display: 'block', textAlign: 'center' }}
         data-ad-layout='in-article'
         data-ad-format='fluid'
-        data-adtest={siteConfig('ADSENSE_GOOGLE_TEST') ? 'on' : 'off'}
-        data-ad-client={siteConfig('ADSENSE_GOOGLE_ID')}
+        data-adtest={ADSENSE_GOOGLE_TEST ? 'on' : 'off'}
+        data-ad-client={ADSENSE_GOOGLE_ID}
         data-ad-slot={siteConfig('ADSENSE_GOOGLE_SLOT_IN_ARTICLE')}></ins>
     )
   }
@@ -134,8 +136,8 @@ const AdSlot = ({ type = 'show' }) => {
         data-ad-format='fluid'
         data-ad-layout-key='-5j+cz+30-f7+bf'
         style={{ display: 'block' }}
-        data-adtest={siteConfig('ADSENSE_GOOGLE_TEST') ? 'on' : 'off'}
-        data-ad-client={siteConfig('ADSENSE_GOOGLE_ID')}
+        data-adtest={ADSENSE_GOOGLE_TEST ? 'on' : 'off'}
+        data-ad-client={ADSENSE_GOOGLE_ID}
         data-ad-slot={siteConfig('ADSENSE_GOOGLE_SLOT_FLOW')}></ins>
     )
   }
@@ -147,8 +149,8 @@ const AdSlot = ({ type = 'show' }) => {
         className='adsbygoogle'
         style={{ display: 'block', textAlign: 'center' }}
         data-ad-format='autorelaxed'
-        data-adtest={siteConfig('ADSENSE_GOOGLE_TEST') ? 'on' : 'off'}
-        data-ad-client={siteConfig('ADSENSE_GOOGLE_ID')}
+        data-adtest={ADSENSE_GOOGLE_TEST ? 'on' : 'off'}
+        data-ad-client={ADSENSE_GOOGLE_ID}
         data-ad-slot={siteConfig('ADSENSE_GOOGLE_SLOT_NATIVE')}></ins>
     )
   }
@@ -158,8 +160,8 @@ const AdSlot = ({ type = 'show' }) => {
     <ins
       className='adsbygoogle'
       style={{ display: 'block' }}
-      data-ad-client={siteConfig('ADSENSE_GOOGLE_ID')}
-      data-adtest={siteConfig('ADSENSE_GOOGLE_TEST') ? 'on' : 'off'}
+      data-ad-client={ADSENSE_GOOGLE_ID}
+      data-adtest={ADSENSE_GOOGLE_TEST ? 'on' : 'off'}
       data-ad-slot={siteConfig('ADSENSE_GOOGLE_SLOT_AUTO')}
       data-ad-format='auto'
       data-full-width-responsive='true'></ins>

lib/config.js

@@ -14,15 +14,39 @@ import { deepClone } from './utils'
  * @param {*} extendConfig ; 参考配置对象{key:val}，如果notion中找不到优先尝试在这里面查找
  * @returns
  */
-export const siteConfig = (key, defaultVal = null, extendConfig = null) => {
-  let global = null
+export const siteConfig = (key, defaultVal = null, extendConfig = {}) => {
+  if (!key) {
+    return null
+  }
+
+  // 特殊配置处理；某些配置只在服务端生效；而Global的NOTION_CONFIG仅限前端组件使用，因此需要从extendConfig中读取
+  switch (key) {
+    case 'NEXT_REVALIDATE_SECOND':
+    case 'POST_RECOMMEND_COUNT':
+    case 'IMAGE_COMPRESS_WIDTH':
+    case 'PSEUDO_STATIC':
+    case 'POSTS_SORT_BY':
+    case 'POSTS_PER_PAGE':
+    case 'POST_PREVIEW_LINES':
+    case 'POST_URL_PREFIX':
+    case 'POST_LIST_STYLE':
+    case 'POST_LIST_PREVIEW':
+    case 'POST_URL_PREFIX_MAPPING_CATEGORY':
+      return convertVal(extendConfig[key] || defaultVal || BLOG[key])
+    default:
+  }
+
+  let global = {}
   try {
-    const isClient = typeof window !== 'undefined'
+    // const isClient = typeof window !== 'undefined'
     // eslint-disable-next-line react-hooks/rules-of-hooks
-    global = isClient ? useGlobal() : {}
+    global = useGlobal()
     // eslint-disable-next-line react-hooks/rules-of-hooks
     // global = useGlobal()
-  } catch (error) {}
+  } catch (error) {
+    // 本地调试用
+    // console.warn('SiteConfig警告', key, error)
+  }
 
   // 首先 配置最优先读取NOTION中的表格配置
   let val = null
@@ -66,6 +90,16 @@ export const siteConfig = (key, defaultVal = null, extendConfig = null) => {
   }
 
   // 从Notion_CONFIG读取的配置通常都是字符串，适当转义
+  return convertVal(val)
+}
+
+/**
+ * 配置默认都是string类型；
+ * 识别配置的值是否数字、布尔、[]数组，若是则转成对应类型
+ * @param {*} val
+ * @returns
+ */
+export const convertVal = val => {
   if (typeof val === 'string') {
     // 解析布尔
     if (val === 'true' || val === 'false') {

lib/db/getSiteData.js

@@ -7,7 +7,7 @@ import { getConfigMapFromConfigPage } from '@/lib/notion/getNotionConfig'
 import getPageProperties, {
   adjustPageProperties
 } from '@/lib/notion/getPageProperties'
-import { getPostBlocks, getSingleBlock } from '@/lib/notion/getPostBlocks'
+import { fetchInBatches, getPostBlocks } from '@/lib/notion/getPostBlocks'
 import { compressImage, mapImgUrl } from '@/lib/notion/mapImage'
 import { deepClone } from '@/lib/utils'
 import { idToUuid } from 'notion-utils'
@@ -236,12 +236,13 @@ function getCategoryOptions(schema) {
  * @param from
  * @returns {Promise<{title,description,pageCover,icon}>}
  */
-function getSiteInfo({ collection, block, NOTION_CONFIG, pageId }) {
-  const defaultTitle = siteConfig('TITLE', '', NOTION_CONFIG)
-  const defaultDescription = siteConfig('DESCRIPTION', '', NOTION_CONFIG)
-  const defaultPageCover = siteConfig('HOME_BANNER_IMAGE', '', NOTION_CONFIG)
-  const defaultIcon = siteConfig('AVATAR', '', NOTION_CONFIG)
-  const defaultLink = siteConfig('LINK', '', NOTION_CONFIG)
+function getSiteInfo({ collection, block, NOTION_CONFIG }) {
+  const defaultTitle = NOTION_CONFIG?.TITLE || BLOG.TITLE
+  const defaultDescription = NOTION_CONFIG?.DESCRIPTION || BLOG.DESCRIPTION
+  const defaultPageCover =
+    NOTION_CONFIG?.HOME_BANNER_IMAGE || BLOG.HOME_BANNER_IMAGE
+  const defaultIcon = NOTION_CONFIG?.AVATAR || BLOG.AVATAR
+  const defaultLink = NOTION_CONFIG?.LINK || BLOG.LINK
   if (!collection && !block) {
     return {
       title: defaultTitle,
@@ -370,13 +371,14 @@ const EmptyData = pageId => {
  * @returns {Promise<JSX.Element|null|*>}
  */
 async function getDataBaseInfoByNotionAPI({ pageId, from }) {
+  console.log('[Fetching Data]', pageId, from)
   const pageRecordMap = await getPostBlocks(pageId, from)
   if (!pageRecordMap) {
     console.error('can`t get Notion Data ; Which id is: ', pageId)
     return {}
   }
   pageId = idToUuid(pageId)
-  const block = pageRecordMap.block || {}
+  let block = pageRecordMap.block || {}
   const rawMetadata = block[pageId]?.value
   // Check Type Page-Database和Inline-Database
   if (
@@ -401,6 +403,7 @@ async function getDataBaseInfoByNotionAPI({ pageId, from }) {
     collectionView,
     viewIds
   )
+
   if (pageIds?.length === 0) {
     console.error(
       '获取到的文章列表为空，请检查notion模板',
@@ -414,29 +417,22 @@ async function getDataBaseInfoByNotionAPI({ pageId, from }) {
     // console.log('有效Page数量', pageIds?.length)
   }
 
-  // 获取每篇文章基础数据
+  // 抓取主数据库最多抓取1000个blocks，溢出的数block这里统一抓取一遍
+  const blockIdsNeedFetch = []
   for (let i = 0; i < pageIds.length; i++) {
     const id = pageIds[i]
     const value = block[id]?.value
     if (!value) {
-      // 如果找不到文章对应的block，说明发生了溢出，使用pageID再去请求
-      const pageBlock = await getSingleBlock(id, from)
-      if (pageBlock.block[id].value) {
-        const properties =
-          (await getPageProperties(
-            id,
-            pageBlock.block[id].value,
-            schema,
-            null,
-            getTagOptions(schema)
-          )) || null
-        if (properties) {
-          collectionData.push(properties)
-        }
-      }
-      continue
+      blockIdsNeedFetch.push(id)
     }
+  }
+  const fetchedBlocks = await fetchInBatches(blockIdsNeedFetch)
+  block = Object.assign({}, block, fetchedBlocks)
 
+  // 获取每篇文章基础数据
+  for (let i = 0; i < pageIds.length; i++) {
+    const id = pageIds[i]
+    const value = block[id]?.value || fetchedBlocks[id]?.value
     const properties =
       (await getPageProperties(
         id,
@@ -445,6 +441,7 @@ async function getDataBaseInfoByNotionAPI({ pageId, from }) {
         null,
         getTagOptions(schema)
       )) || null
+
     if (properties) {
       collectionData.push(properties)
     }

lib/notion/getPageProperties.js

@@ -186,7 +186,7 @@ export function adjustPageProperties(properties, NOTION_CONFIG) {
   }
 
   // 开启伪静态路径
-  if (siteConfig('PSEUDO_STATIC', false, NOTION_CONFIG)) {
+  if (JSON.parse(NOTION_CONFIG?.PSEUDO_STATIC || BLOG.PSEUDO_STATIC)) {
     if (
       !properties?.href?.endsWith('.html') &&
       !properties?.href?.startsWith('http')

lib/notion/getPostBlocks.js

@@ -1,6 +1,6 @@
 import BLOG from '@/blog.config'
-import { NotionAPI } from 'notion-client'
 import { getDataFromCache, setDataToCache } from '@/lib/cache/cache_manager'
+import { NotionAPI } from 'notion-client'
 import { deepClone, delay } from '../utils'
 
 /**
@@ -41,7 +41,7 @@ export async function getSingleBlock(id, from) {
     return pageBlock
   }
 
-  pageBlock = await getPageWithRetry(id, from)
+  pageBlock = await getPageWithRetry(id, 'single_' + from)
 
   if (pageBlock) {
     await setDataToCache(cacheKey, pageBlock)
@@ -56,10 +56,18 @@ export async function getSingleBlock(id, from) {
  */
 export async function getPageWithRetry(id, from, retryAttempts = 3) {
   if (retryAttempts && retryAttempts > 0) {
-    console.log('[API-->>请求]', `from:${from}`, `id:${id}`, retryAttempts < 3 ? `剩余重试次数:${retryAttempts}` : '')
+    console.log(
+      '[API-->>请求]',
+      `from:${from}`,
+      `id:${id}`,
+      retryAttempts < 3 ? `剩余重试次数:${retryAttempts}` : ''
+    )
     try {
       const authToken = BLOG.NOTION_ACCESS_TOKEN || null
-      const api = new NotionAPI({ authToken, userTimeZone: Intl.DateTimeFormat().resolvedOptions().timeZone })
+      const api = new NotionAPI({
+        authToken,
+        userTimeZone: Intl.DateTimeFormat().resolvedOptions().timeZone
+      })
       const start = new Date().getTime()
       const pageData = await api.getPage(id)
       const end = new Date().getTime()
@@ -125,9 +133,13 @@ function filterPostBlocks(id, blockMap, slice) {
     }
 
     // 如果是文件，或嵌入式PDF，需要重新加密签名
-    if ((b?.value?.type === 'file' || b?.value?.type === 'pdf' || b?.value?.type === 'video' || b?.value?.type === 'audio') &&
-         b?.value?.properties?.source?.[0][0] &&
-         b?.value?.properties?.source?.[0][0].indexOf('amazonaws.com') > 0
+    if (
+      (b?.value?.type === 'file' ||
+        b?.value?.type === 'pdf' ||
+        b?.value?.type === 'video' ||
+        b?.value?.type === 'audio') &&
+      b?.value?.properties?.source?.[0][0] &&
+      b?.value?.properties?.source?.[0][0].indexOf('amazonaws.com') > 0
     ) {
       const oldUrl = b?.value?.properties?.source?.[0][0]
       const newUrl = `https://notion.so/signed/${encodeURIComponent(oldUrl)}?table=block&id=${b?.value?.id}`
@@ -141,3 +153,38 @@ function filterPostBlocks(id, blockMap, slice) {
   }
   return clonePageBlock
 }
+
+/**
+ * 根据[]ids，批量抓取blocks
+ * 在获取数据库文章列表时，超过一定数量的block会被丢弃，因此根据pageId批量抓取block
+ * @param {*} ids
+ * @param {*} batchSize
+ * @returns
+ */
+export const fetchInBatches = async (ids, batchSize = 100) => {
+  const authToken = BLOG.NOTION_ACCESS_TOKEN || null
+  const api = new NotionAPI({
+    authToken,
+    userTimeZone: Intl.DateTimeFormat().resolvedOptions().timeZone
+  })
+
+  let fetchedBlocks = {}
+  for (let i = 0; i < ids.length; i += batchSize) {
+    const batch = ids.slice(i, i + batchSize)
+    console.log('[API-->>请求] Fetching missing blocks', ids.length)
+    const start = new Date().getTime()
+    const pageChunk = await api.getBlocks(batch)
+    const end = new Date().getTime()
+    console.log(
+      `[API<<--响应] 耗时:${end - start}ms Fetching missing blocks count:${ids.length} `
+    )
+
+    console.log('[API<<--响应]')
+    fetchedBlocks = Object.assign(
+      {},
+      fetchedBlocks,
+      pageChunk?.recordMap?.block
+    )
+  }
+  return fetchedBlocks
+}

lib/robots.txt.js

@@ -1,8 +1,8 @@
 import fs from 'fs'
-import { siteConfig } from './config'
 
-export async function generateRobotsTxt(NOTION_CONFIG) {
-  const LINK = siteConfig('LINK', '', NOTION_CONFIG)
+export async function generateRobotsTxt(props) {
+  const { siteInfo } = props
+  const LINK = siteInfo?.link
   const content = `
     # *
     User-agent: *