fix: Fix devops pipeline run logs bugs (#1900)

* refactor: refactor pipeline run time Signed-off-by: yazhou <[email protected]> * fix: fix kubeconfigFile error Signed-off-by: yazhou <[email protected]> * fix: fix pipeline step log too big Signed-off-by: yazhou <[email protected]> * fix: limit pipeline run logs Signed-off-by: yazhou <[email protected]> --------- Signed-off-by: yazhou <[email protected]>
kubesphere · Oct 23, 2023 · 62bcf69 · 62bcf69
1 parent a7f948d
commit 62bcf69
Show file tree

Hide file tree

Showing 6 changed files with 206 additions and 32 deletions.
diff --git a/src/pages/devops/components/Pipeline/StepModals/params.jsx b/src/pages/devops/components/Pipeline/StepModals/params.jsx
@@ -84,8 +84,8 @@ const setCredentialType = str => {
   if (type) {
     const credentialType = Object.entries(typesDict).find(
       typeArr => typeArr[1] === type
-    )
-    return credentialType ? credentialType[0] : null
+    )?.[0]
+    return credentialType
   }
   return null
 }

diff --git a/src/pages/devops/containers/Pipelines/Detail/PipelineLogDialog/FullLogs/index.jsx b/src/pages/devops/containers/Pipelines/Detail/PipelineLogDialog/FullLogs/index.jsx
@@ -43,6 +43,9 @@ export default class FullLogs extends React.Component {
 
   @computed
   get isLogFinish() {
+    if (this.store.overflow) {
+      return true
+    }
     const logs = this.store.runDetailLogs.split('\n')
     let index = logs.length - 1
     let start = 0

diff --git a/src/pages/devops/containers/Pipelines/Detail/PipelineLogDialog/Timer.jsx b/src/pages/devops/containers/Pipelines/Detail/PipelineLogDialog/Timer.jsx
@@ -0,0 +1,50 @@
+/*
+ * This file is part of KubeSphere Console.
+ * Copyright (C) 2019 The KubeSphere Console Authors.
+ *
+ * KubeSphere Console is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * KubeSphere Console is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with KubeSphere Console.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { useEffect, useRef, useState } from 'react'
+import { formatUsedTime } from 'utils/index'
+
+const TimeCounter = ({ startTime, time }) => {
+  const [seconds, setSeconds] = useState(0)
+
+  const timerRef = useRef(null)
+
+  useEffect(() => {
+    if (startTime) {
+      const interval = setInterval(() => {
+        const endTime = new Date()
+        const diff = endTime.getTime() - new Date(startTime).getTime()
+        setSeconds(diff)
+      }, 300)
+      timerRef.current = interval
+      return () => clearInterval(interval)
+    }
+  }, [startTime])
+
+  useEffect(() => {
+    if (time && timerRef.current) {
+      clearInterval(timerRef.current)
+    }
+  }, [time])
+
+  return t('DURATION_VALUE', {
+    value: startTime ? formatUsedTime(time ?? seconds) : '-',
+  })
+}
+
+export default TimeCounter
diff --git a/src/pages/devops/containers/Pipelines/Detail/PipelineLogDialog/index.jsx b/src/pages/devops/containers/Pipelines/Detail/PipelineLogDialog/index.jsx
@@ -18,16 +18,16 @@
 
 import React from 'react'
 import classNames from 'classnames'
-import { isEmpty, isArray } from 'lodash'
-import { action, observable, computed, toJS, reaction } from 'mobx'
+import { isArray, isEmpty } from 'lodash'
+import { action, computed, observable, reaction, toJS } from 'mobx'
 import { observer } from 'mobx-react'
 import { Modal } from 'components/Base'
 import { Button } from '@kube-design/components'
 import Status from 'devops/components/Status'
 import { getPipelineStatus } from 'utils/status'
-import { formatUsedTime } from 'utils'
 import RunStore from 'stores/devops/run'
 
+import TimeCounter from 'devops/containers/Pipelines/Detail/PipelineLogDialog/Timer'
 import LogItem from './logItem'
 import styles from './index.scss'
 import FullLogs from './FullLogs'
@@ -215,8 +215,7 @@ export default class PipelineLog extends React.Component {
     //   )
     // }
 
-    const time = this.activeStage?.durationInMillis ?? ''
-
+    // const time = this.activeStage?.durationInMillis ?? ''
     return (
       <>
         <div className={styles.container}>
@@ -226,9 +225,14 @@ export default class PipelineLog extends React.Component {
           <div className={styles.right}>
             <div className={styles.header}>
               <span>
-                {t('DURATION_VALUE', {
-                  value: time ? formatUsedTime(time) : '-',
-                })}
+                <TimeCounter
+                  startTime={this.activeStage?.startTime}
+                  time={
+                    this.activeStage?.result !== 'UNKNOWN'
+                      ? this.activeStage?.durationInMillis
+                      : undefined
+                  }
+                />
               </span>
               <Button onClick={this.handleVisableLog}>
                 {t('VIEW_FULL_LOG')}

diff --git a/src/stores/devops/log.js b/src/stores/devops/log.js
@@ -29,6 +29,11 @@ export default class PipelineRunStore extends BaseStore {
   }
 
   async getStepLog({ devops, cluster, name, branch, runId, nodeId, stepId }) {
+    const headers = branch
+      ? {}
+      : {
+          'x-file-size-limit': 1024 * 1024 * 5,
+        }
     const result = await request.defaults({
       url: `${this.getDevopsUrlV2({
         cluster,
@@ -37,18 +42,32 @@ export default class PipelineRunStore extends BaseStore {
         branch ? `/branches/${encodeURIComponent(branch)}` : ''
       }/runs/${runId}/nodes/${nodeId}/steps/${stepId}/log/?start=${this
         .stepLogData.start || 0}`,
+      options: { headers },
       handler: resp => {
         if (resp.status === 200) {
           return resp.text().then(res => ({ data: res, headers: resp.headers }))
         }
       },
     })
-    const prevLog = this.stepLogData.log
+    const prevLog = !this.stepLogData.start ? '' : this.stepLogData.log
     this.stepLogData = {
       log: prevLog + get(result, 'data', ''),
       start: result.headers.get('x-text-size'),
       hasMore: Boolean(result.headers.get('x-more-data')),
     }
+    if (
+      result.headers.get('X-File-Size-Limit-Out') === 'true' ||
+      this.log?.length > 1024 * 1024 * 5
+    ) {
+      this.stepLogData.hasMore = false
+      this.stepLogData.log += `\n
+*****************************************************************   
+*                                                               * 
+* The log is too large, please download it to view the details. *
+*                                                               * 
+*****************************************************************    
+      `
+    }
   }
 
   @action

diff --git a/src/stores/devops/run.js b/src/stores/devops/run.js
@@ -20,7 +20,7 @@ import { omit, isArray, get, isEmpty } from 'lodash'
 import { saveAs } from 'file-saver'
 import { action, observable, toJS } from 'mobx'
 import { Notify } from '@kube-design/components'
-import { safeParseJSON } from 'utils'
+import { getClusterUrl, safeParseJSON } from 'utils'
 
 import BaseStore from '../devops'
 
@@ -76,7 +76,21 @@ export default class PipelineRunStore extends BaseStore {
 
   // entire run log
   @observable
-  runDetailLogs = ''
+  runStartDetailLogs = ''
+
+  @observable
+  logSize = 0
+
+  @observable
+  overflow = false
+
+  @observable
+  hasMore = false
+
+  hasMoreNotify = false
+
+  @observable
+  lastDetailLogs = ''
 
   getUrl({ cluster, devops, name }) {
     return `${this.getDevopsUrlV2({
@@ -304,30 +318,114 @@ export default class PipelineRunStore extends BaseStore {
   }
 
   @action
-  async getRunStatusLogs({ devops, name, branch, runId, cluster }) {
-    // TODO: use response headers offset
-    const result = await request.get(
-      `${this.getRunUrl({
+  async getRunStatusLogs(
+    { devops, name, branch, runId, cluster },
+    refresh = false
+  ) {
+    if (refresh) {
+      this.logSize = 0
+      this.runStartDetailLogs = ''
+      this.hasMore = false
+    }
+    if (this.overflow) {
+      const result = await request.get(
+        `${this.getRunUrl({
+          cluster,
+          devops,
+          name,
+          branch,
+          runId,
+        })}log/?thresholdInKB=150`
+      )
+      this.hasMore = true
+      this.lastDetailLogs = `
+
+*****************************************************************    
+* The log is too large, please download it to view the details. *
+*                                                               * 
+* The following is the latest 150KB log.                        * 
+*****************************************************************    
+
+${result}`
+    } else {
+      const start = this.logSize
+      const result = await request.get(
+        `${this.getRunUrl({
+          cluster,
+          devops,
+          name,
+          branch,
+          runId,
+        })}log/?start=${this.logSize}`,
+        {},
+        {
+          headers: {
+            'x-file-size-limit': 1024 * 1024 * 10,
+            'x-with-headers': true,
+          },
+        }
+      )
+      const size = result.headers.get('x-text-size')
+      if (size) {
+        this.logSize = Number(size)
+        this.hasMore = Boolean(result.headers.get('x-more-data'))
+      } else {
+        this.logSize += Number(result.headers.get('x-text-size'))
+        this.hasMore = Boolean(result.headers.get('x-more-data'))
+      }
+      this.overflow =
+        Boolean(result.headers.get('X-File-Size-Limit-Out')) ||
+        this.logSize >= 1024 * 1024 * 10
+
+      result.text().then(text => {
+        if (start === 0) {
+          this.runStartDetailLogs = text
+          return
+        }
+        // console.log(this.runStartDetailLogs.slice(-100).split('\n'))
+        const arr = this.runStartDetailLogs.slice(-100).split('\n')
+        if (arr.length >= 2) {
+          arr.pop()
+          if (arr.length && arr.pop().startsWith('Finished:')) {
+            //
+          } else {
+            this.runStartDetailLogs += text
+          }
+        } else {
+          this.runStartDetailLogs += text
+        }
+      })
+    }
+  }
+  // removeSameWords = (str1, str2) => {
+  //   const end = str1.slice(-100)
+  //   const index = str2.indexOf(end)
+  //   if (index === -1) {
+  //     return str2
+  //   }
+  //   return str2.slice(index + end.length)
+  // }
+
+  async handleDownloadLogs({ devops, name, branch, cluster }) {
+    name = decodeURIComponent(name)
+    const url = getClusterUrl(
+      `${window.location.protocol}//${window.location.host}/${this.getRunUrl({
         cluster,
         devops,
         name,
         branch,
-        runId,
-      })}log/?start=0`
+        runId: this.runDetail.id,
+      })}log/?start=0&download=true`
     )
-    this.runDetailLogs = result
-  }
-
-  async handleDownloadLogs({ devops, name, branch, cluster }) {
-    name = decodeURIComponent(name)
-    await this.getRunStatusLogs({
-      devops,
-      name,
-      branch,
-      runId: this.runDetail.id,
-      cluster,
-    })
-    this.saveAsFile(this.runDetailLogs, 'log.txt')
+    const a = document.createElement('a')
+    a.href = url
+    a.download = `${name}-${this.runDetail.id}-${this.runDetail.name}.log`
+    a.headers = {
+      'x-add-res-header': JSON.stringify({
+        'Content-Disposition': `attachment; filename=${name}-${this.runDetail.name}.log`,
+      }),
+    }
+    a.click()
   }
 
   saveAsFile = (text = '', fileName = 'default.txt') => {