From 22d9bf47d397794dff92cd5c6eb14f5189a21549 Mon Sep 17 00:00:00 2001 From: Ivan Date: Mon, 1 Mar 2021 18:14:13 +0800 Subject: [PATCH] =?UTF-8?q?feat(pipeline-tasks):=20=E6=B5=81=E6=B0=B4?= =?UTF-8?q?=E7=BA=BF=E4=BA=BA=E7=89=A9=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/ci-cd.md | 33 ++++++++++ src/app.module.ts | 2 + .../dtos/create-pipeline-task.input.ts | 11 ++++ .../enums/pipeline-units.enum.ts | 7 +++ .../enums/task-statuses.enum.ts | 6 ++ .../models/pipeline-task-logs.model.ts | 9 +++ src/pipeline-tasks/pipeline-task.entity.ts | 33 ++++++++++ .../pipeline-tasks.constants.ts | 1 + src/pipeline-tasks/pipeline-tasks.module.ts | 13 ++++ .../pipeline-tasks.resolver.spec.ts | 18 ++++++ src/pipeline-tasks/pipeline-tasks.resolver.ts | 4 ++ .../pipeline-tasks.service.spec.ts | 26 ++++++++ src/pipeline-tasks/pipeline-tasks.service.ts | 60 +++++++++++++++++++ 13 files changed, 223 insertions(+) create mode 100644 docs/ci-cd.md create mode 100644 src/pipeline-tasks/dtos/create-pipeline-task.input.ts create mode 100644 src/pipeline-tasks/enums/pipeline-units.enum.ts create mode 100644 src/pipeline-tasks/enums/task-statuses.enum.ts create mode 100644 src/pipeline-tasks/models/pipeline-task-logs.model.ts create mode 100644 src/pipeline-tasks/pipeline-task.entity.ts create mode 100644 src/pipeline-tasks/pipeline-tasks.constants.ts create mode 100644 src/pipeline-tasks/pipeline-tasks.module.ts create mode 100644 src/pipeline-tasks/pipeline-tasks.resolver.spec.ts create mode 100644 src/pipeline-tasks/pipeline-tasks.resolver.ts create mode 100644 src/pipeline-tasks/pipeline-tasks.service.spec.ts create mode 100644 src/pipeline-tasks/pipeline-tasks.service.ts diff --git a/docs/ci-cd.md b/docs/ci-cd.md new file mode 100644 index 0000000..5cd3bdf --- /dev/null +++ b/docs/ci-cd.md @@ -0,0 +1,33 @@ +# CI/CD 流程 +0. 准备 + - project information + - commit hash +1. checkout +2. install dependencies +3. run test script +5. run deploy script +6. clear workspace + +## 流水线任务单元描述 +```json +{ + "version": 1, + "unit": { + "install-dependencies": { + "script": "npm ci" + }, + "test": { + "script": "npm test" + }, + "build": { + "script": "npm build" + }, + "deploy": { + "script": [ + "npm build" + ] + } + } +} + +``` \ No newline at end of file diff --git a/src/app.module.ts b/src/app.module.ts index d77d6c8..b6c5144 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -8,6 +8,7 @@ import { AppService } from './app.service'; import { ProjectsModule } from './projects/projects.module'; import { ReposModule } from './repos/repos.module'; import { PipelinesModule } from './pipelines/pipelines.module'; +import { PipelineTasksModule } from './pipeline-tasks/pipeline-tasks.module'; import configuration from './commons/config/configuration'; @Module({ @@ -41,6 +42,7 @@ import configuration from './commons/config/configuration'; ProjectsModule, ReposModule, PipelinesModule, + PipelineTasksModule, ], controllers: [AppController], providers: [AppService, AppResolver], diff --git a/src/pipeline-tasks/dtos/create-pipeline-task.input.ts b/src/pipeline-tasks/dtos/create-pipeline-task.input.ts new file mode 100644 index 0000000..340d50e --- /dev/null +++ b/src/pipeline-tasks/dtos/create-pipeline-task.input.ts @@ -0,0 +1,11 @@ +import { InputType } from '@nestjs/graphql'; +import { PipelineUnits } from '../enums/pipeline-units.enum'; + +@InputType() +export class CreatePipelineTaskInput { + pipelineId: string; + + commit: string; + + units: PipelineUnits[]; +} diff --git a/src/pipeline-tasks/enums/pipeline-units.enum.ts b/src/pipeline-tasks/enums/pipeline-units.enum.ts new file mode 100644 index 0000000..4c85a25 --- /dev/null +++ b/src/pipeline-tasks/enums/pipeline-units.enum.ts @@ -0,0 +1,7 @@ +export enum PipelineUnits { + checkout = 'checkout', + installDependencies = 'installDependencies', + test = 'test', + deploy = 'deploy', + cleanUp = 'cleanUp', +} diff --git a/src/pipeline-tasks/enums/task-statuses.enum.ts b/src/pipeline-tasks/enums/task-statuses.enum.ts new file mode 100644 index 0000000..1d11115 --- /dev/null +++ b/src/pipeline-tasks/enums/task-statuses.enum.ts @@ -0,0 +1,6 @@ +export enum TaskStatuses { + success = 'success', + failed = 'failed', + working = 'working', + pending = 'pending', +} diff --git a/src/pipeline-tasks/models/pipeline-task-logs.model.ts b/src/pipeline-tasks/models/pipeline-task-logs.model.ts new file mode 100644 index 0000000..7178b73 --- /dev/null +++ b/src/pipeline-tasks/models/pipeline-task-logs.model.ts @@ -0,0 +1,9 @@ +import { TaskStatuses } from '../enums/task-statuses.enum'; +import { PipelineUnits } from '../enums/pipeline-units.enum'; +export class PipelineTaskLogs { + unit: PipelineUnits; + status: TaskStatuses; + startedAt?: Date; + endedAt?: Date; + logs: string[]; +} diff --git a/src/pipeline-tasks/pipeline-task.entity.ts b/src/pipeline-tasks/pipeline-task.entity.ts new file mode 100644 index 0000000..50716c1 --- /dev/null +++ b/src/pipeline-tasks/pipeline-task.entity.ts @@ -0,0 +1,33 @@ +import { ObjectType } from '@nestjs/graphql'; +import { Column, Entity, ManyToOne } from 'typeorm'; +import { Pipeline } from '../pipelines/pipeline.entity'; +import { PipelineTaskLogs } from './models/pipeline-task-logs.model'; +import { TaskStatuses } from './enums/task-statuses.enum'; +import { PipelineUnits } from './enums/pipeline-units.enum'; + +@ObjectType() +@Entity() +export class PipelineTask { + @ManyToOne(() => Pipeline) + pipeline: Pipeline; + @Column() + pipelineId: string; + + @Column() + commit: string; + + @Column({ type: 'enum', enum: PipelineUnits, array: true }) + units: PipelineUnits[]; + + @Column({ type: 'jsonb', default: '[]' }) + logs: PipelineTaskLogs[]; + + @Column({ type: 'enum', enum: TaskStatuses, default: TaskStatuses.pending }) + status: TaskStatuses; + + @Column() + startedAt: Date; + + @Column() + endedAt: Date; +} diff --git a/src/pipeline-tasks/pipeline-tasks.constants.ts b/src/pipeline-tasks/pipeline-tasks.constants.ts new file mode 100644 index 0000000..08e8f67 --- /dev/null +++ b/src/pipeline-tasks/pipeline-tasks.constants.ts @@ -0,0 +1 @@ +export const PIPELINE_TASK_QUEUE = 'PIPELINE_TASK_QUEUE'; diff --git a/src/pipeline-tasks/pipeline-tasks.module.ts b/src/pipeline-tasks/pipeline-tasks.module.ts new file mode 100644 index 0000000..f10fd99 --- /dev/null +++ b/src/pipeline-tasks/pipeline-tasks.module.ts @@ -0,0 +1,13 @@ +import { Module } from '@nestjs/common'; +import { PipelineTasksService } from './pipeline-tasks.service'; +import { PipelineTasksResolver } from './pipeline-tasks.resolver'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { PipelineTask } from './pipeline-task.entity'; +import { Pipeline } from '../pipelines/pipeline.entity'; +import { BullModule } from '@nestjs/bull'; + +@Module({ + imports: [TypeOrmModule.forFeature([PipelineTask, Pipeline])], + providers: [PipelineTasksService, PipelineTasksResolver], +}) +export class PipelineTasksModule {} diff --git a/src/pipeline-tasks/pipeline-tasks.resolver.spec.ts b/src/pipeline-tasks/pipeline-tasks.resolver.spec.ts new file mode 100644 index 0000000..8eeebb8 --- /dev/null +++ b/src/pipeline-tasks/pipeline-tasks.resolver.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { PipelineTasksResolver } from './pipeline-tasks.resolver'; + +describe('PipelineTasksResolver', () => { + let resolver: PipelineTasksResolver; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [PipelineTasksResolver], + }).compile(); + + resolver = module.get(PipelineTasksResolver); + }); + + it('should be defined', () => { + expect(resolver).toBeDefined(); + }); +}); diff --git a/src/pipeline-tasks/pipeline-tasks.resolver.ts b/src/pipeline-tasks/pipeline-tasks.resolver.ts new file mode 100644 index 0000000..78286e6 --- /dev/null +++ b/src/pipeline-tasks/pipeline-tasks.resolver.ts @@ -0,0 +1,4 @@ +import { Resolver } from '@nestjs/graphql'; + +@Resolver() +export class PipelineTasksResolver {} diff --git a/src/pipeline-tasks/pipeline-tasks.service.spec.ts b/src/pipeline-tasks/pipeline-tasks.service.spec.ts new file mode 100644 index 0000000..01b8202 --- /dev/null +++ b/src/pipeline-tasks/pipeline-tasks.service.spec.ts @@ -0,0 +1,26 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { PipelineTasksService } from './pipeline-tasks.service'; +import { getRepositoryToken } from '@nestjs/typeorm'; +import { PipelineTask } from './pipeline-task.entity'; + +describe('PipelineTasksService', () => { + let service: PipelineTasksService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + PipelineTasksService, + { + provide: getRepositoryToken(PipelineTask), + useValue: {}, + }, + ], + }).compile(); + + service = module.get(PipelineTasksService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/src/pipeline-tasks/pipeline-tasks.service.ts b/src/pipeline-tasks/pipeline-tasks.service.ts new file mode 100644 index 0000000..05b97ec --- /dev/null +++ b/src/pipeline-tasks/pipeline-tasks.service.ts @@ -0,0 +1,60 @@ +import { Injectable } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; +import { PipelineTask } from './pipeline-task.entity'; +import { Repository } from 'typeorm'; +import { CreatePipelineTaskInput } from './dtos/create-pipeline-task.input'; +import { RedisService } from 'nestjs-redis'; +import { Pipeline } from '../pipelines/pipeline.entity'; +import { InjectQueue } from '@nestjs/bull'; +import { PIPELINE_TASK_QUEUE } from './pipeline-tasks.constants'; +import { Queue } from 'bull'; + +@Injectable() +export class PipelineTasksService { + constructor( + @InjectRepository(PipelineTask) + private readonly repository: Repository, + @InjectRepository(Pipeline) + private readonly pipelineRepository: Repository, + private readonly redis: RedisService, + @InjectQueue(PIPELINE_TASK_QUEUE) + private readonly queue: Queue, + ) {} + async addTask(dto: CreatePipelineTaskInput) { + const pipeline = await this.pipelineRepository.findOneOrFail({ + where: { id: dto.pipelineId }, + relations: ['project'], + }); + const task = await this.repository.save(this.repository.create(dto)); + task.pipeline = pipeline; + + const [lckKey, tasksKey] = this.getRedisTokens(pipeline); + const redis = this.redis.getClient(); + await redis.set(lckKey, 0, 'EX', 10, 'NX'); + const lckSemaphore = await redis.incr(lckKey); + if (lckSemaphore > 1) { + await this.redis + .getClient() + .lpush(tasksKey, JSON.stringify(task)) + .finally(() => { + return redis.decr(lckKey); + }); + } else { + this.queue.add(task); + } + } + + async doTask(task: PipelineTask) { + const tasksKey = this.getRedisTokens(task.pipeline)[1]; + + const redis = this.redis.getClient(); + const nextTask = await redis.rpop(tasksKey); + if (nextTask) { + this.doTask(task).then(); + } + } + + getRedisTokens(pipeline: Pipeline): [string, string] { + return [`pipeline-${pipeline.id}:lck`, `pipeline-${pipeline.id}:tasks`]; + } +}