feat-pipelines #1

Merged
Ivan merged 25 commits from feat-pipelines into master 2021-03-24 20:50:41 +08:00
13 changed files with 223 additions and 0 deletions
Showing only changes of commit 22d9bf47d3 - Show all commits

33
docs/ci-cd.md Normal file
View File

@ -0,0 +1,33 @@
# CI/CD 流程
0. 准备
- project information
- commit hash
1. checkout
2. install dependencies
3. run test script
5. run deploy script
6. clear workspace
## 流水线任务单元描述
```json
{
"version": 1,
"unit": {
"install-dependencies": {
"script": "npm ci"
},
"test": {
"script": "npm test"
},
"build": {
"script": "npm build"
},
"deploy": {
"script": [
"npm build"
]
}
}
}
```

View File

@ -8,6 +8,7 @@ import { AppService } from './app.service';
import { ProjectsModule } from './projects/projects.module';
import { ReposModule } from './repos/repos.module';
import { PipelinesModule } from './pipelines/pipelines.module';
import { PipelineTasksModule } from './pipeline-tasks/pipeline-tasks.module';
import configuration from './commons/config/configuration';
@Module({
@ -41,6 +42,7 @@ import configuration from './commons/config/configuration';
ProjectsModule,
ReposModule,
PipelinesModule,
PipelineTasksModule,
],
controllers: [AppController],
providers: [AppService, AppResolver],

View File

@ -0,0 +1,11 @@
import { InputType } from '@nestjs/graphql';
import { PipelineUnits } from '../enums/pipeline-units.enum';
@InputType()
export class CreatePipelineTaskInput {
pipelineId: string;
commit: string;
units: PipelineUnits[];
}

View File

@ -0,0 +1,7 @@
export enum PipelineUnits {
checkout = 'checkout',
installDependencies = 'installDependencies',
test = 'test',
deploy = 'deploy',
cleanUp = 'cleanUp',
}

View File

@ -0,0 +1,6 @@
export enum TaskStatuses {
success = 'success',
failed = 'failed',
working = 'working',
pending = 'pending',
}

View File

@ -0,0 +1,9 @@
import { TaskStatuses } from '../enums/task-statuses.enum';
import { PipelineUnits } from '../enums/pipeline-units.enum';
export class PipelineTaskLogs {
unit: PipelineUnits;
status: TaskStatuses;
startedAt?: Date;
endedAt?: Date;
logs: string[];
}

View File

@ -0,0 +1,33 @@
import { ObjectType } from '@nestjs/graphql';
import { Column, Entity, ManyToOne } from 'typeorm';
import { Pipeline } from '../pipelines/pipeline.entity';
import { PipelineTaskLogs } from './models/pipeline-task-logs.model';
import { TaskStatuses } from './enums/task-statuses.enum';
import { PipelineUnits } from './enums/pipeline-units.enum';
@ObjectType()
@Entity()
export class PipelineTask {
@ManyToOne(() => Pipeline)
pipeline: Pipeline;
@Column()
pipelineId: string;
@Column()
commit: string;
@Column({ type: 'enum', enum: PipelineUnits, array: true })
units: PipelineUnits[];
@Column({ type: 'jsonb', default: '[]' })
logs: PipelineTaskLogs[];
@Column({ type: 'enum', enum: TaskStatuses, default: TaskStatuses.pending })
status: TaskStatuses;
@Column()
startedAt: Date;
@Column()
endedAt: Date;
}

View File

@ -0,0 +1 @@
export const PIPELINE_TASK_QUEUE = 'PIPELINE_TASK_QUEUE';

View File

@ -0,0 +1,13 @@
import { Module } from '@nestjs/common';
import { PipelineTasksService } from './pipeline-tasks.service';
import { PipelineTasksResolver } from './pipeline-tasks.resolver';
import { TypeOrmModule } from '@nestjs/typeorm';
import { PipelineTask } from './pipeline-task.entity';
import { Pipeline } from '../pipelines/pipeline.entity';
import { BullModule } from '@nestjs/bull';
@Module({
imports: [TypeOrmModule.forFeature([PipelineTask, Pipeline])],
providers: [PipelineTasksService, PipelineTasksResolver],
})
export class PipelineTasksModule {}

View File

@ -0,0 +1,18 @@
import { Test, TestingModule } from '@nestjs/testing';
import { PipelineTasksResolver } from './pipeline-tasks.resolver';
describe('PipelineTasksResolver', () => {
let resolver: PipelineTasksResolver;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [PipelineTasksResolver],
}).compile();
resolver = module.get<PipelineTasksResolver>(PipelineTasksResolver);
});
it('should be defined', () => {
expect(resolver).toBeDefined();
});
});

View File

@ -0,0 +1,4 @@
import { Resolver } from '@nestjs/graphql';
@Resolver()
export class PipelineTasksResolver {}

View File

@ -0,0 +1,26 @@
import { Test, TestingModule } from '@nestjs/testing';
import { PipelineTasksService } from './pipeline-tasks.service';
import { getRepositoryToken } from '@nestjs/typeorm';
import { PipelineTask } from './pipeline-task.entity';
describe('PipelineTasksService', () => {
let service: PipelineTasksService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
PipelineTasksService,
{
provide: getRepositoryToken(PipelineTask),
useValue: {},
},
],
}).compile();
service = module.get<PipelineTasksService>(PipelineTasksService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
});

View File

@ -0,0 +1,60 @@
import { Injectable } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { PipelineTask } from './pipeline-task.entity';
import { Repository } from 'typeorm';
import { CreatePipelineTaskInput } from './dtos/create-pipeline-task.input';
import { RedisService } from 'nestjs-redis';
import { Pipeline } from '../pipelines/pipeline.entity';
import { InjectQueue } from '@nestjs/bull';
import { PIPELINE_TASK_QUEUE } from './pipeline-tasks.constants';
import { Queue } from 'bull';
@Injectable()
export class PipelineTasksService {
constructor(
@InjectRepository(PipelineTask)
private readonly repository: Repository<PipelineTask>,
@InjectRepository(Pipeline)
private readonly pipelineRepository: Repository<Pipeline>,
private readonly redis: RedisService,
@InjectQueue(PIPELINE_TASK_QUEUE)
private readonly queue: Queue<PipelineTask>,
) {}
async addTask(dto: CreatePipelineTaskInput) {
const pipeline = await this.pipelineRepository.findOneOrFail({
where: { id: dto.pipelineId },
relations: ['project'],
});
const task = await this.repository.save(this.repository.create(dto));
task.pipeline = pipeline;
const [lckKey, tasksKey] = this.getRedisTokens(pipeline);
const redis = this.redis.getClient();
await redis.set(lckKey, 0, 'EX', 10, 'NX');
const lckSemaphore = await redis.incr(lckKey);
if (lckSemaphore > 1) {
await this.redis
.getClient()
.lpush(tasksKey, JSON.stringify(task))
.finally(() => {
return redis.decr(lckKey);
});
} else {
this.queue.add(task);
}
}
async doTask(task: PipelineTask) {
const tasksKey = this.getRedisTokens(task.pipeline)[1];
const redis = this.redis.getClient();
const nextTask = await redis.rpop(tasksKey);
if (nextTask) {
this.doTask(task).then();
}
}
getRedisTokens(pipeline: Pipeline): [string, string] {
return [`pipeline-${pipeline.id}:lck`, `pipeline-${pipeline.id}:tasks`];
}
}