fix: Fix devops pipeline too large run logs

locales/en/l10n-devopsProjects-pipeline-details.js

@@ -235,6 +235,7 @@ module.exports = {
   // detail page // run log // task status
   RUN_LOGS: 'Run Logs',
   VIEW_FULL_LOG: 'View Full Logs',
+  VIEW_REAL_TIME_LOG: 'View Real-time Logs',
   // detail page // run log // task status // pipeline log modal
   PIPELINE_LOG: 'Pipeline Logs',
   // detail page // Create Pipeline modal // add step modal

locales/es/l10n-devopsProjects-pipeline-details.js

@@ -224,6 +224,7 @@ module.exports = {
   // detail page // run log // task status
   RUN_LOGS: 'Run Logs',
   VIEW_FULL_LOG: 'View Full Logs',
+  VIEW_REAL_TIME_LOG: 'View Real-time Logs',
   // detail page // run log // task status // pipeline log modal
   PIPELINE_LOG: 'Pipeline Logs',
   // detail page // Create Pipeline modal // add step modal

locales/tc/l10n-devopsProjects-pipeline-details.js

@@ -220,6 +220,7 @@ module.exports = {
   // detail page // run log // task status
   RUN_LOGS: 'Run Logs',
   VIEW_FULL_LOG: 'View Full Logs',
+  VIEW_REAL_TIME_LOG: 'View Real-time Logs',
   // detail page // run log // task status // pipeline log modal
   PIPELINE_LOG: 'Pipeline Logs',
   // detail page // Create Pipeline modal // add step modal

locales/zh/l10n-devopsProjects-pipeline-details.js

@@ -219,6 +219,7 @@ module.exports = {
   // detail page // run log // task status
   RUN_LOGS: '运行日志',
   VIEW_FULL_LOG: '查看完整日志',
+  VIEW_REAL_TIME_LOG: '查看实时日志',
   // detail page // run log // task status // pipeline log modal
   PIPELINE_LOG: '流水线日志',
   // detail page // Create Pipeline modal // add step modal

src/pages/devops/components/Pipeline/StepModals/params.jsx

@@ -84,8 +84,8 @@ const setCredentialType = str => {
   if (type) {
     const credentialType = Object.entries(typesDict).find(
       typeArr => typeArr[1] === type
-    )
-    return credentialType ? credentialType[0] : null
+    )?.[0]
+    return credentialType
   }
   return null
 }

src/pages/devops/containers/Pipelines/Detail/PipelineLogDialog/FullLogs/index.jsx

@@ -43,6 +43,9 @@ export default class FullLogs extends React.Component {
 
   @computed
   get isLogFinish() {
+    if (this.store.overflow) {
+      return true
+    }
     const logs = this.store.runDetailLogs.split('\n')
     let index = logs.length - 1
     let start = 0

src/pages/devops/containers/Pipelines/Detail/PipelineLogDialog/Timer.jsx

@@ -0,0 +1,50 @@
+/*
+ * This file is part of KubeSphere Console.
+ * Copyright (C) 2019 The KubeSphere Console Authors.
+ *
+ * KubeSphere Console is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * KubeSphere Console is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with KubeSphere Console.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { useEffect, useRef, useState } from 'react'
+import { formatUsedTime } from 'utils/index'
+
+const TimeCounter = ({ startTime, time }) => {
+  const [seconds, setSeconds] = useState(0)
+
+  const timerRef = useRef(null)
+
+  useEffect(() => {
+    if (startTime) {
+      const interval = setInterval(() => {
+        const endTime = new Date()
+        const diff = endTime.getTime() - new Date(startTime).getTime()
+        setSeconds(diff)
+      }, 300)
+      timerRef.current = interval
+      return () => clearInterval(interval)
+    }
+  }, [startTime])
+
+  useEffect(() => {
+    if (time && timerRef.current) {
+      clearInterval(timerRef.current)
+    }
+  }, [time])
+
+  return t('DURATION_VALUE', {
+    value: startTime ? formatUsedTime(time ?? seconds) : '-',
+  })
+}
+
+export default TimeCounter

src/pages/devops/containers/Pipelines/Detail/PipelineLogDialog/index.jsx

@@ -18,16 +18,16 @@
 
 import React from 'react'
 import classNames from 'classnames'
-import { isEmpty, isArray } from 'lodash'
-import { action, observable, computed, toJS, reaction } from 'mobx'
+import { isArray, isEmpty } from 'lodash'
+import { action, computed, observable, reaction, toJS } from 'mobx'
 import { observer } from 'mobx-react'
 import { Modal } from 'components/Base'
 import { Button } from '@kube-design/components'
 import Status from 'devops/components/Status'
 import { getPipelineStatus } from 'utils/status'
-import { formatUsedTime } from 'utils'
 import RunStore from 'stores/devops/run'
 
+import TimeCounter from 'devops/containers/Pipelines/Detail/PipelineLogDialog/Timer'
 import LogItem from './logItem'
 import styles from './index.scss'
 import FullLogs from './FullLogs'
@@ -40,6 +40,9 @@ export default class PipelineLog extends React.Component {
      * @type {RunStore}
      */
     this.store = new RunStore()
+    this.state = {
+      isDownloading: false,
+    }
 
     this.reaction = reaction(
       () => this.isEmptySteps,
@@ -126,6 +129,16 @@ export default class PipelineLog extends React.Component {
   //   this.refreshFlag = !this.refreshFlag
   // }, 1000)
 
+  handleDownload = async () => {
+    this.setState({ isDownloading: true })
+    await this.store.handleDownloadLogs(this.props.params)
+    this.setState({ isDownloading: false })
+  }
+
+  handleJumpFullLogs = () => {
+    this.store.handleJumpFullLogs(this.props.params)
+  }
+
   renderLeftTab(stage, index) {
     if (Array.isArray(stage)) {
       return (
@@ -200,6 +213,28 @@ export default class PipelineLog extends React.Component {
     ))
   }
 
+  get isRunning() {
+    return this.activeStage?.result && this.activeStage?.result === 'UNKNOWN'
+  }
+
+  renderLogButton = () => {
+    if (this.isRunning) {
+      return (
+        <Button onClick={this.handleVisableLog}>
+          {t('VIEW_REAL_TIME_LOG')}
+        </Button>
+      )
+    }
+    return (
+      <span>
+        <Button loading={this.state.isLoading} onClick={this.handleDownload}>
+          {t('DOWNLOAD_LOGS')}
+        </Button>
+        <Button onClick={this.handleJumpFullLogs}>{t('VIEW_FULL_LOG')}</Button>
+      </span>
+    )
+  }
+
   render() {
     const { nodes } = this.props
     const _nodes = toJS(nodes)
@@ -215,8 +250,7 @@ export default class PipelineLog extends React.Component {
     //   )
     // }
 
-    const time = this.activeStage?.durationInMillis ?? ''
-
+    // const time = this.activeStage?.durationInMillis ?? ''
     return (
       <>
         <div className={styles.container}>
@@ -226,13 +260,16 @@ export default class PipelineLog extends React.Component {
           <div className={styles.right}>
             <div className={styles.header}>
               <span>
-                {t('DURATION_VALUE', {
-                  value: time ? formatUsedTime(time) : '-',
-                })}
+                <TimeCounter
+                  startTime={this.activeStage?.startTime}
+                  time={
+                    this.activeStage?.result !== 'UNKNOWN'
+                      ? this.activeStage?.durationInMillis
+                      : undefined
+                  }
+                />
               </span>
-              <Button onClick={this.handleVisableLog}>
-                {t('VIEW_FULL_LOG')}
-              </Button>
+              {this.renderLogButton()}
             </div>
             <div className={styles.logContainer}>{this.renderLogContent()}</div>
           </div>

src/stores/devops/log.js

@@ -29,26 +29,47 @@ export default class PipelineRunStore extends BaseStore {
   }
 
   async getStepLog({ devops, cluster, name, branch, runId, nodeId, stepId }) {
+    const params = this.stepLogData.start
+      ? `?start=${this.stepLogData.start}`
+      : ''
+    const headers = branch
+      ? {}
+      : {
+          'x-file-size-limit': 1024 * 1024 * 5,
+        }
     const result = await request.defaults({
       url: `${this.getDevopsUrlV2({
         cluster,
         devops,
       })}pipelines/${decodeURIComponent(name)}${
         branch ? `/branches/${encodeURIComponent(branch)}` : ''
-      }/runs/${runId}/nodes/${nodeId}/steps/${stepId}/log/?start=${this
-        .stepLogData.start || 0}`,
+      }/runs/${runId}/nodes/${nodeId}/steps/${stepId}/log/${params}`,
+      options: { headers },
       handler: resp => {
         if (resp.status === 200) {
           return resp.text().then(res => ({ data: res, headers: resp.headers }))
         }
       },
     })
-    const prevLog = this.stepLogData.log
+    const prevLog = !this.stepLogData.start ? '' : this.stepLogData.log
     this.stepLogData = {
       log: prevLog + get(result, 'data', ''),
       start: result.headers.get('x-text-size'),
       hasMore: Boolean(result.headers.get('x-more-data')),
     }
+    if (
+      result.headers.get('X-File-Size-Limit-Out') === 'true' ||
+      this.log?.length > 1024 * 1024 * 5
+    ) {
+      this.stepLogData.hasMore = false
+      this.stepLogData.log += `\n
+*****************************************************************   
+*                                                               * 
+* The log is too large, please download it to view the details. *
+*                                                               * 
+*****************************************************************    
+      `
+    }
   }
 
   @action

src/stores/devops/run.js

@@ -81,6 +81,9 @@ export default class PipelineRunStore extends BaseStore {
   @observable
   logSize = 0
 
+  @observable
+  overflow = false
+
   @observable
   hasMore = false
 
@@ -329,7 +332,7 @@ export default class PipelineRunStore extends BaseStore {
       this.runStartDetailLogs = ''
       this.hasMore = false
     }
-    if (this.logSize >= 1024 * 1024 * 20) {
+    if (this.overflow) {
       const result = await request.get(
         `${this.getRunUrl({
           cluster,
@@ -350,40 +353,77 @@ export default class PipelineRunStore extends BaseStore {
 
 ${result}`
     } else {
+      const start = this.logSize
+      const params = start ? `?start=${start}` : ''
       const result = await request.get(
         `${this.getRunUrl({
           cluster,
           devops,
           name,
           branch,
           runId,
-        })}log/?start=${this.logSize}`,
+        })}log/${params}`,
         {},
         {
           headers: {
-            'x-file-size-limit': 1024 * 100,
+            // 'x-file-size-limit': 1024 * 1024 * 10,
             'x-with-headers': true,
           },
         }
       )
-      this.logSize += Number(result.headers.get('X-File-Size'))
-      this.hasMore = result.headers.get('X-File-Size-Limit-Out') !== 'true'
+      const size = result.headers.get('x-text-size')
+      if (size) {
+        this.logSize = Number(size)
+        this.hasMore = Boolean(result.headers.get('x-more-data'))
+      } else {
+        this.logSize += Number(result.headers.get('x-text-size'))
+        this.hasMore = Boolean(result.headers.get('x-more-data'))
+      }
+      this.overflow =
+        Boolean(result.headers.get('X-File-Size-Limit-Out')) ||
+        this.logSize >= 1024 * 1024 * 10
+
       result.text().then(text => {
-        this.runStartDetailLogs += this.removeSameWords(
-          this.runStartDetailLogs,
-          text
-        )
+        if (start === 0) {
+          this.runStartDetailLogs = text
+          return
+        }
+        // console.log(this.runStartDetailLogs.slice(-100).split('\n'))
+        const arr = this.runStartDetailLogs.slice(-100).split('\n')
+        if (arr.length >= 2) {
+          arr.pop()
+          if (arr.length && arr.pop().startsWith('Finished:')) {
+            //
+          } else {
+            this.runStartDetailLogs += text
+          }
+        } else {
+          this.runStartDetailLogs += text
+        }
       })
     }
   }
-
-  removeSameWords = (str1, str2) => {
-    const end = str1.slice(-100)
-    const index = str2.indexOf(end)
-    if (index === -1) {
-      return str2
-    }
-    return str2.slice(index + end.length)
+  // removeSameWords = (str1, str2) => {
+  //   const end = str1.slice(-100)
+  //   const index = str2.indexOf(end)
+  //   if (index === -1) {
+  //     return str2
+  //   }
+  //   return str2.slice(index + end.length)
+  // }
+
+  handleJumpFullLogs({ devops, name, branch, cluster }) {
+    name = decodeURIComponent(name)
+    const url = getClusterUrl(
+      `${window.location.protocol}//${window.location.host}/${this.getRunUrl({
+        cluster,
+        devops,
+        name,
+        branch,
+        runId: this.runDetail.id,
+      })}log/?start=0`
+    )
+    window.open(url)
   }
 
   async handleDownloadLogs({ devops, name, branch, cluster }) {
@@ -470,4 +510,4 @@ ${result}`
       }
     )
   }
-}
+}