Skip to content

Commit

Permalink
refactoring job-log view => ajax
Browse files Browse the repository at this point in the history
ansibleguy committed Feb 10, 2024
1 parent cdbb36a commit bf22d75
Showing 9 changed files with 323 additions and 85 deletions.
4 changes: 3 additions & 1 deletion src/ansible-webui/aw/api.py
Original file line number Diff line number Diff line change
@@ -2,7 +2,8 @@
from drf_spectacular.views import SpectacularAPIView, SpectacularSwaggerView

from aw.api_endpoints.key import APIKey, APIKeyItem
from aw.api_endpoints.job import APIJob, APIJobItem, APIJobExecutionItem, APIJobExecutionLogs, APIJobExecutionLogFile
from aw.api_endpoints.job import APIJob, APIJobItem, APIJobExecutionItem, APIJobExecutionLogs, \
APIJobExecutionLogFile, APIJobExecution
from aw.api_endpoints.permission import APIPermission, APIPermissionItem
from aw.api_endpoints.filesystem import APIFsBrowse

@@ -13,6 +14,7 @@
path('api/job/<int:job_id>/<int:exec_id>/log', APIJobExecutionLogFile.as_view()),
path('api/job/<int:job_id>/<int:exec_id>', APIJobExecutionItem.as_view()),
path('api/job/<int:job_id>', APIJobItem.as_view()),
path('api/job_exec', APIJobExecution.as_view()),
path('api/job', APIJob.as_view()),
path('api/permission/<int:perm_id>', APIPermissionItem.as_view()),
path('api/permission', APIPermission.as_view()),
173 changes: 149 additions & 24 deletions src/ansible-webui/aw/api_endpoints/job.py
Original file line number Diff line number Diff line change
@@ -1,21 +1,23 @@
from django.core.exceptions import ObjectDoesNotExist
from django.db.utils import IntegrityError
from django.shortcuts import HttpResponse
from rest_framework.views import APIView
from rest_framework import serializers
from rest_framework.response import Response
from drf_spectacular.utils import extend_schema, OpenApiResponse
from drf_spectacular.utils import extend_schema, OpenApiResponse, OpenApiParameter

from aw.config.hardcoded import JOB_EXECUTION_LIMIT
from aw.model.job import Job, JobExecution, BaseJobCredentials, \
CHOICE_JOB_PERMISSION_READ, CHOICE_JOB_PERMISSION_WRITE, CHOICE_JOB_PERMISSION_EXECUTE, CHOICE_JOB_PERMISSION_FULL
CHOICE_JOB_PERMISSION_READ, CHOICE_JOB_PERMISSION_WRITE, CHOICE_JOB_PERMISSION_EXECUTE, \
CHOICE_JOB_PERMISSION_FULL
from aw.api_endpoints.base import API_PERMISSION, get_api_user, BaseResponse, GenericResponse
from aw.api_endpoints.job_util import get_viewable_jobs_serialized, JobReadResponse
from aw.api_endpoints.job_util import get_viewable_jobs_serialized, JobReadResponse, get_job_executions_serialized, \
JobExecutionReadResponse, get_viewable_jobs, get_job_execution_serialized
from aw.utils.permission import has_job_permission
from aw.execute.queue import queue_add
from aw.execute.util import update_execution_status, is_execution_status
from aw.utils.util import is_null

LIMIT_JOB_RESULTS = 10


class JobWriteRequest(serializers.ModelSerializer):
class Meta:
@@ -46,6 +48,27 @@ def _find_job_and_execution(job_id: int, exec_id: int) -> tuple[Job, (JobExecuti
return job, None


def _job_execution_count(request) -> (int, None):
max_count = None
if 'execution_count' in request.GET:
max_count = int(request.GET['execution_count'])
max_count = min(max_count, 1000)

return max_count


def _want_job_executions(request) -> tuple:
max_count = None
if 'executions' in request.GET and request.GET['executions'] == 'true':
try:
return True, _job_execution_count(request)

except TypeError:
pass

return False, max_count


class APIJob(APIView):
http_method_names = ['post', 'get']
serializer_class = JobReadResponse
@@ -58,10 +81,32 @@ class APIJob(APIView):
200: OpenApiResponse(JobReadResponse, description='Return list of jobs'),
},
summary='Return list of all jobs the current user is privileged to view.',
operation_id='job_list'
operation_id='job_list',
parameters=[
OpenApiParameter(
name='executions', type=bool, default=False, description='Return list of job-executions',
required=False,
),
OpenApiParameter(
name='execution_count', type=int, default=JOB_EXECUTION_LIMIT,
description='Maximum count of job-executions to return',
required=False,
),
],
)
def get(request):
return Response(data=get_viewable_jobs_serialized(get_api_user(request)), status=200)
want_exec, exec_count = _want_job_executions(request)
if want_exec:
data = get_viewable_jobs_serialized(
user=get_api_user(request),
executions=True,
execution_count=exec_count,
)

else:
data = get_viewable_jobs_serialized(get_api_user(request))

return Response(data=data, status=200)

@extend_schema(
request=JobWriteRequest,
@@ -112,11 +157,23 @@ class APIJobItem(APIView):
request=None,
responses={
200: OpenApiResponse(JobReadResponse, description='Return job information'),
400: OpenApiResponse(GenericResponse, description='Bad parameters provided'),
403: OpenApiResponse(GenericResponse, description='Not privileged to view the job'),
404: OpenApiResponse(JobReadResponse, description='Job does not exist'),
},
summary='Return information about a job.',
operation_id='job_view'
operation_id='job_view',
parameters=[
OpenApiParameter(
name='executions', type=bool, default=False, description='Return list of job-executions',
required=False,
),
OpenApiParameter(
name='execution_count', type=int, default=JOB_EXECUTION_LIMIT,
description='Maximum count of job-executions to return',
required=False,
),
],
)
def get(self, request, job_id: int):
self.serializer_class = JobReadResponse
@@ -128,7 +185,13 @@ def get(self, request, job_id: int):
if not has_job_permission(user=user, job=job, permission_needed=CHOICE_JOB_PERMISSION_READ):
return Response(data={'msg': f"Job '{job.name}' is not viewable"}, status=403)

return Response(data=JobReadResponse(instance=job).data, status=200)
data = JobReadResponse(instance=job).data

want_exec, exec_count = _want_job_executions(request)
if want_exec:
data['executions'] = get_job_executions_serialized(job=job, execution_count=exec_count)

return Response(data=data, status=200)

@extend_schema(
request=None,
@@ -286,9 +349,9 @@ class APIJobExecutionLogs(APIView):
@extend_schema(
request=None,
responses={
200: OpenApiResponse(JobReadResponse, description='Return job logs'),
403: OpenApiResponse(JobReadResponse, description='Not privileged to view the job logs'),
404: OpenApiResponse(JobReadResponse, description='Job, execution or logs do not exist'),
200: OpenApiResponse(JobExecutionLogReadResponse, description='Return job logs'),
403: OpenApiResponse(JobExecutionLogReadResponse, description='Not privileged to view the job logs'),
404: OpenApiResponse(JobExecutionLogReadResponse, description='Job, execution or log-file do not exist'),
},
summary='Get logs of a job execution.',
operation_id='job_exec_logs'
@@ -309,10 +372,13 @@ def get(self, request, job_id: int, exec_id: int, line_start: int = 0):
lines = logfile.readlines()
return Response(data={'lines': lines[line_start:]}, status=200)

except ObjectDoesNotExist:
except (ObjectDoesNotExist, FileNotFoundError):
pass

return Response(data={'msg': f"Job with ID '{job_id}' or execution does not exist"}, status=404)
return Response(
data={'msg': f"Job with ID '{job_id}', execution with ID '{exec_id}' or log-file does not exist"},
status=404,
)


class APIJobExecutionLogFile(APIView):
@@ -323,12 +389,19 @@ class APIJobExecutionLogFile(APIView):
@extend_schema(
request=None,
responses={
200: OpenApiResponse(JobReadResponse, description='Download job log-file'),
403: OpenApiResponse(JobReadResponse, description='Not privileged to view the job logs'),
404: OpenApiResponse(JobReadResponse, description='Job, execution or logs do not exist'),
200: OpenApiResponse(GenericResponse, description='Download job log-file'),
403: OpenApiResponse(GenericResponse, description='Not privileged to view the job logs'),
404: OpenApiResponse(GenericResponse, description='Job, execution or log-file do not exist'),
},
summary='Download log-file of a job execution.',
operation_id='job_exec_logfile'
operation_id='job_exec_logfile',
parameters=[
OpenApiParameter(
name='type', type=str, default='stdout',
description="Type of log-file to download. Either 'stdout' or 'stderr'",
required=False,
),
],
)
def get(self, request, job_id: int, exec_id: int):
user = get_api_user(request)
@@ -339,15 +412,67 @@ def get(self, request, job_id: int, exec_id: int):
if not has_job_permission(user=user, job=job, permission_needed=CHOICE_JOB_PERMISSION_READ):
return Response(data={'msg': f"Not privileged to view logs of the job '{job.name}'"}, status=403)

if execution.log_stdout is None:
logfile = execution.log_stdout
if 'type' in request.GET:
logfile = execution.log_stderr if request.GET['type'] == 'stderr' else logfile

if logfile is None:
return Response(data={'msg': f"No logs found for job '{job.name}'"}, status=404)

with open(execution.log_stdout, 'rb') as logfile:
response = Response(logfile.read(), content_type='text/plain', status=200)
response['Content-Disposition'] = f"inline; filename={execution.log_stdout.rsplit('/', 1)[1]}"
with open(logfile, 'rb') as _logfile:
content_b = _logfile.read()
if content_b == b'':
return Response(data={'msg': f"Job log-file is empty: '{logfile}'"}, status=404)

response = HttpResponse(content_b, content_type='text/plain', status=200)
response['Content-Disposition'] = f"inline; filename={logfile.rsplit('/', 1)[1]}"
return response

except ObjectDoesNotExist:
except (ObjectDoesNotExist, FileNotFoundError):
pass

return Response(data={'msg': f"Job with ID '{job_id}' or execution does not exist"}, status=404)
return Response(
data={'msg': f"Job with ID '{job_id}', execution with ID '{exec_id}' or log-file does not exist"},
status=404,
)


class APIJobExecution(APIView):
http_method_names = ['get']
serializer_class = JobExecutionReadResponse
permission_classes = API_PERMISSION

@extend_schema(
request=None,
responses={
200: OpenApiResponse(JobExecutionReadResponse, description='Return job-execution information'),
404: OpenApiResponse(JobExecutionReadResponse, description='No viewable jobs or executions found'),
},
summary='Return list of job-executions the current user is privileged to view.',
operation_id='job_exec_list',
parameters=[
OpenApiParameter(
name='execution_count', type=int, default=JOB_EXECUTION_LIMIT,
description='Maximum count of job-executions to return',
required=False,
),
],
)
def get(self, request):
# pylint: disable=E1101
jobs = get_viewable_jobs(get_api_user(request))
if len(jobs) == 0:
return Response(data={'msg': 'No viewable jobs found'}, status=404)

exec_count = _job_execution_count(request)
if exec_count is None:
exec_count = JOB_EXECUTION_LIMIT

serialized = []
for execution in JobExecution.objects.filter(job__in=jobs).order_by('updated')[:exec_count]:
serialized.append(get_job_execution_serialized(execution))

if len(serialized) == 0:
return Response(data={'msg': 'No viewable job-executions found'}, status=404)

return Response(data=serialized, status=200)
80 changes: 77 additions & 3 deletions src/ansible-webui/aw/api_endpoints/job_util.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
from django.conf import settings
from rest_framework import serializers

from aw.model.job import Job
from aw.config.hardcoded import SHORT_TIME_FORMAT, JOB_EXECUTION_LIMIT
from aw.model.job import Job, CHOICES_JOB_EXEC_STATUS, JobExecution
from aw.utils.permission import get_viewable_jobs
from aw.utils.util import datetime_from_db


class JobReadResponse(serializers.ModelSerializer):
@@ -11,5 +13,77 @@ class Meta:
fields = Job.api_fields_read


def get_viewable_jobs_serialized(user: settings.AUTH_USER_MODEL) -> list[dict]:
return [JobReadResponse(instance=job).data for job in get_viewable_jobs(user)]
class JobExecutionReadResponse(serializers.ModelSerializer):
class Meta:
model = JobExecution
fields = JobExecution.api_fields_read

job_name = serializers.CharField(required=False)
job_comment = serializers.CharField(required=False)
user_name = serializers.CharField(required=False)
status_name = serializers.CharField(required=False)
failed = serializers.BooleanField(required=False)
error_s = serializers.CharField(required=False)
error_m = serializers.CharField(required=False)
time_start = serializers.CharField(required=False)
time_fin = serializers.CharField(required=False)
log_stdout = serializers.CharField(required=False)
log_stdout_url = serializers.CharField(required=False)
log_stderr = serializers.CharField(required=False)
log_stderr_url = serializers.CharField(required=False)


def get_job_execution_serialized(execution: JobExecution) -> dict:
# pylint: disable=E1101
serialized = {
'id': execution.id,
'job': execution.job.id,
'job_name': execution.job.name,
'job_comment': execution.job.comment,
'user': execution.user.id,
'user_name': execution.user.username if execution.user is not None else 'Scheduled',
'status': execution.status,
'status_name': CHOICES_JOB_EXEC_STATUS[execution.status][1],
'time_start': datetime_from_db(execution.created).strftime(SHORT_TIME_FORMAT),
'time_fin': None,
'failed': None,
'error_s': None,
'error_m': None,
'log_stdout': execution.log_stdout,
'log_stdout_url': f"/api/job/{execution.job.id}/{execution.id}/log",
'log_stderr': execution.log_stderr,
'log_stderr_url': f"/api/job/{execution.job.id}/{execution.id}/log?type=stderr",
}
if execution.result is not None:
serialized['time_fin'] = datetime_from_db(execution.result.time_fin).strftime(SHORT_TIME_FORMAT)
serialized['failed'] = execution.result.failed
if execution.result.error is not None:
serialized['error_s'] = execution.result.error.short
serialized['error_m'] = execution.result.error.med

return serialized


def get_job_executions_serialized(job: Job, execution_count: int = JOB_EXECUTION_LIMIT) -> list[dict]:
# pylint: disable=E1101
serialized = []
for execution in JobExecution.objects.filter(job=job).order_by('-updated')[:execution_count]:
serialized.append(get_job_execution_serialized(execution))

return serialized


def get_viewable_jobs_serialized(
user: settings.AUTH_USER_MODEL, executions: bool = False,
execution_count: int = None
) -> list[dict]:
serialized = []

for job in get_viewable_jobs(user):
job_serialized = JobReadResponse(instance=job).data
if executions:
job_serialized['executions'] = get_job_executions_serialized(job=job, execution_count=execution_count)

serialized.append(job_serialized)

return serialized
1 change: 1 addition & 0 deletions src/ansible-webui/aw/config/hardcoded.py
Original file line number Diff line number Diff line change
@@ -9,3 +9,4 @@
FILE_TIME_FORMAT = '%Y-%m-%d_%H-%M-%S'
KEY_TIME_FORMAT = '%Y-%m-%d-%H-%M-%S'
MIN_SECRET_LEN = 30
JOB_EXECUTION_LIMIT = 20
7 changes: 6 additions & 1 deletion src/ansible-webui/aw/model/job.py
Original file line number Diff line number Diff line change
@@ -162,7 +162,7 @@ class Job(BaseJob):
form_fields = CHANGE_FIELDS
api_fields_read = ['id']
api_fields_read.extend(CHANGE_FIELDS)
api_fields_write = api_fields_read
api_fields_write = api_fields_read.copy()
api_fields_write.extend(['vault_pass', 'become_pass', 'connect_pass'])

name = models.CharField(max_length=150)
@@ -328,6 +328,11 @@ def __str__(self) -> str:


class JobExecution(BaseJob):
api_fields_read = [
'id', 'job', 'job_name', 'user', 'user_name', 'result', 'status', 'status_name', 'time_start', 'time_fin',
'failed', 'error_s', 'error_m', 'log_stdout', 'log_stdout_url', 'log_stderr', 'log_stderr_url',
]

# NOTE: scheduled execution will have no user
user = models.ForeignKey(
settings.AUTH_USER_MODEL, on_delete=models.SET_NULL, null=True,
Loading

0 comments on commit bf22d75

Please sign in to comment.