feat(pipeline-task): 流水线任务日志订阅接口。

This commit is contained in:
Ivan Li 2021-03-15 13:30:52 +08:00
parent aa92c518f0
commit 4e7c825170
12 changed files with 15803 additions and 23 deletions

View File

@ -2,6 +2,8 @@
"cSpell.words": [ "cSpell.words": [
"Repos", "Repos",
"lpush", "lpush",
"rpop" "lrange",
"rpop",
"rpush"
] ]
} }

15696
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -40,6 +40,7 @@
"graphql-tools": "^7.0.2", "graphql-tools": "^7.0.2",
"js-yaml": "^4.0.0", "js-yaml": "^4.0.0",
"nestjs-redis": "^1.2.8", "nestjs-redis": "^1.2.8",
"observable-to-async-generator": "^1.0.1-rc",
"pg": "^8.5.1", "pg": "^8.5.1",
"ramda": "^0.27.1", "ramda": "^0.27.1",
"reflect-metadata": "^0.1.13", "reflect-metadata": "^0.1.13",

View File

@ -0,0 +1,8 @@
import { ArgsType } from '@nestjs/graphql';
import { IsUUID } from 'class-validator';
@ArgsType()
export class PipelineTaskLogArgs {
@IsUUID()
taskId: string;
}

View File

@ -1,7 +1,13 @@
import { PipelineTask } from './../pipeline-task.entity'; import { PipelineTask } from './../pipeline-task.entity';
import { PipelineUnits } from '../enums/pipeline-units.enum';
import { Field, HideField, ObjectType } from '@nestjs/graphql';
@ObjectType()
export class PipelineTaskLogMessage { export class PipelineTaskLogMessage {
@HideField()
task: PipelineTask; task: PipelineTask;
@Field(() => PipelineUnits)
unit: PipelineUnits;
time: Date; time: Date;
message: string; message: string;

View File

@ -0,0 +1,25 @@
import { Test, TestingModule } from '@nestjs/testing';
import { PipelineTaskLogsService } from './pipeline-task-logs.service';
import { RedisService } from 'nestjs-redis';
describe('PipelineTaskLogsService', () => {
let service: PipelineTaskLogsService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
PipelineTaskLogsService,
{
provide: RedisService,
useValue: {},
},
],
}).compile();
service = module.get<PipelineTaskLogsService>(PipelineTaskLogsService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});

View File

@ -0,0 +1,50 @@
import { Injectable } from '@nestjs/common';
import { PubSub } from 'graphql-subscriptions';
import { RedisService } from 'nestjs-redis';
import { omit } from 'ramda';
import { PipelineTaskLogMessage } from './models/pipeline-task-log-message.module';
import { PipelineTask } from './pipeline-task.entity';
const LOG_TIMEOUT_SECONDS = 10_000;
@Injectable()
export class PipelineTaskLogsService {
constructor(private readonly redisService: RedisService) {}
pubSub = new PubSub();
get redis() {
return this.redisService.getClient();
}
getKeys(task: PipelineTask) {
return `ptl:${task.id}`;
}
async recordLog(log: PipelineTaskLogMessage) {
const logDto = omit(['task'], log);
await Promise.all([
this.pubSub.publish(this.getKeys(log.task), logDto),
this.redis
.expire(this.getKeys(log.task), LOG_TIMEOUT_SECONDS)
.then(() =>
this.redis.rpush(this.getKeys(log.task), JSON.stringify(logDto)),
),
]);
}
async readLogs(task: PipelineTask): Promise<PipelineTaskLogMessage[]> {
return await this.redis.lrange(this.getKeys(task), 0, -1).then((items) =>
items.map((item) => {
const log = JSON.parse(item) as PipelineTaskLogMessage;
log.task = task;
log.time = new Date(log.time);
return log;
}),
);
}
watchLogs(task: PipelineTask) {
return this.pubSub.asyncIterator(this.getKeys(task));
}
}

View File

@ -18,13 +18,13 @@ import { ApplicationException } from '../commons/exceptions/application.exceptio
import { PipelineUnits } from './enums/pipeline-units.enum'; import { PipelineUnits } from './enums/pipeline-units.enum';
import { PipelineTaskLogMessage } from './models/pipeline-task-log-message.module'; import { PipelineTaskLogMessage } from './models/pipeline-task-log-message.module';
import { TaskStatuses } from './enums/task-statuses.enum'; import { TaskStatuses } from './enums/task-statuses.enum';
import { PipelineTaskLogsService } from './pipeline-task-logs.service';
@Processor(PIPELINE_TASK_QUEUE) @Processor(PIPELINE_TASK_QUEUE)
export class PipelineTaskConsumer { export class PipelineTaskConsumer {
constructor( constructor(
private readonly service: PipelineTasksService, private readonly service: PipelineTasksService,
private readonly reposService: ReposService, private readonly reposService: ReposService,
@InjectQueue(PIPELINE_TASK_LOG_QUEUE) private readonly logsService: PipelineTaskLogsService,
private readonly logQueue: Queue<PipelineTaskLogMessage>,
) {} ) {}
@Process() @Process()
async doTask({ data: task, update }: Job<PipelineTask>) { async doTask({ data: task, update }: Job<PipelineTask>) {
@ -68,14 +68,14 @@ export class PipelineTaskConsumer {
} finally { } finally {
unitLog.endedAt = new Date(); unitLog.endedAt = new Date();
task.logs.push(unitLog); task.logs.push(unitLog);
update(task); // await update(task);
} }
} }
} catch (err) { } catch (err) {
console.log(err); console.log(err);
} finally { } finally {
task = await this.service.updateTask(task); task = await this.service.updateTask(task);
update(task); await update(task);
} }
} }
@ -95,12 +95,12 @@ export class PipelineTaskConsumer {
const str = data.toString(); const str = data.toString();
errorMessages.push(str); errorMessages.push(str);
logs.push(str); logs.push(str);
this.logQueue.add(PipelineTaskLogMessage.create(task, str)); this.logsService.recordLog(PipelineTaskLogMessage.create(task, str));
}); });
sub.stdout.on('data', (data: Buffer) => { sub.stdout.on('data', (data: Buffer) => {
const str = data.toString(); const str = data.toString();
logs.push(str); logs.push(str);
this.logQueue.add(PipelineTaskLogMessage.create(task, str)); this.logsService.recordLog(PipelineTaskLogMessage.create(task, str));
}); });
sub.addListener('close', (code) => { sub.addListener('close', (code) => {
if (code === 0) { if (code === 0) {

View File

@ -1,2 +1,3 @@
export const PIPELINE_TASK_QUEUE = 'PIPELINE_TASK_QUEUE'; export const PIPELINE_TASK_QUEUE = 'PIPELINE_TASK_QUEUE';
export const PIPELINE_TASK_LOG_QUEUE = 'PIPELINE_TASK_LOG_QUEUE'; export const PIPELINE_TASK_LOG_QUEUE = 'PIPELINE_TASK_LOG_QUEUE';
export const PIPELINE_TASK_LOG_PUBSUB = 'PIPELINE_TASK_LOG_PUBSUB';

View File

@ -11,7 +11,10 @@ import { PipelineTaskConsumer } from './pipeline-task.consumer';
import { import {
PIPELINE_TASK_QUEUE, PIPELINE_TASK_QUEUE,
PIPELINE_TASK_LOG_QUEUE, PIPELINE_TASK_LOG_QUEUE,
PIPELINE_TASK_LOG_PUBSUB,
} from './pipeline-tasks.constants'; } from './pipeline-tasks.constants';
import { PipelineTaskLogsService } from './pipeline-task-logs.service';
import { PubSub } from 'apollo-server-express';
@Module({ @Module({
imports: [ imports: [
@ -29,6 +32,11 @@ import {
PipelineTasksService, PipelineTasksService,
PipelineTasksResolver, PipelineTasksResolver,
PipelineTaskConsumer, PipelineTaskConsumer,
PipelineTaskLogsService,
{
provide: Symbol(PIPELINE_TASK_LOG_PUBSUB),
useValue: new PubSub(),
},
], ],
}) })
export class PipelineTasksModule {} export class PipelineTasksModule {}

View File

@ -4,10 +4,14 @@ import { PipelineTasksService } from './pipeline-tasks.service';
import { CreatePipelineTaskInput } from './dtos/create-pipeline-task.input'; import { CreatePipelineTaskInput } from './dtos/create-pipeline-task.input';
import { PipelineTaskLogMessage } from './models/pipeline-task-log-message.module'; import { PipelineTaskLogMessage } from './models/pipeline-task-log-message.module';
import { PipelineTaskLogArgs } from './dtos/pipeline-task-log.args'; import { PipelineTaskLogArgs } from './dtos/pipeline-task-log.args';
import { PipelineTaskLogsService } from './pipeline-task-logs.service';
@Resolver() @Resolver()
export class PipelineTasksResolver { export class PipelineTasksResolver {
constructor(private readonly service: PipelineTasksService) {} constructor(
private readonly service: PipelineTasksService,
private readonly logsService: PipelineTaskLogsService,
) {}
@Mutation(() => PipelineTask) @Mutation(() => PipelineTask)
async createPipelineTask(@Args('task') taskDto: CreatePipelineTaskInput) { async createPipelineTask(@Args('task') taskDto: CreatePipelineTaskInput) {
@ -15,4 +19,9 @@ export class PipelineTasksResolver {
} }
@Subscription(() => PipelineTaskLogMessage) @Subscription(() => PipelineTaskLogMessage)
async pipelineTaskLog(@Args() args: PipelineTaskLogArgs) {
const task = await this.service.findTaskById(args.taskId);
const asyncIterator = this.logsService.watchLogs(task);
return asyncIterator;
}
} }

View File

@ -36,6 +36,10 @@ export class PipelineTasksService {
return task; return task;
} }
async findTaskById(id: string) {
return await this.repository.findOneOrFail({ id });
}
async doNextTask(pipeline: Pipeline) { async doNextTask(pipeline: Pipeline) {
const [lckKey, tasksKey] = this.getRedisTokens(pipeline); const [lckKey, tasksKey] = this.getRedisTokens(pipeline);
const redis = this.redis.getClient(); const redis = this.redis.getClient();