Compare commits

..

3 Commits

Author SHA1 Message Date
Ivan
691a3dc377 feat(webhooks): add gitea webhooks api. 2021-03-27 21:46:03 +08:00
Ivan
429de1eaed test(pipeline-tasks): pass test cases. 2021-03-27 15:52:21 +08:00
Ivan
08e5c7e7d3 refactor(pipeline-tasks): remove unused queue. 2021-03-27 15:34:08 +08:00
24 changed files with 8052 additions and 8702 deletions

View File

@ -1,8 +1,11 @@
{
"cSpell.words": [
"Repos",
"boardcat",
"gitea",
"lpush",
"lrange",
"metatype",
"rpop",
"rpush"
]

16132
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -33,6 +33,7 @@
"@types/ramda": "^0.27.38",
"apollo-server-express": "^2.19.2",
"bcrypt": "^5.0.0",
"body-parser": "^1.19.0",
"bull": "^3.20.1",
"class-transformer": "^0.3.2",
"class-validator": "^0.13.1",
@ -51,9 +52,10 @@
"typeorm": "^0.2.30"
},
"devDependencies": {
"@nestjs/cli": "^7.5.1",
"@nestjs/cli": "^7.5.7",
"@nestjs/schematics": "^7.1.3",
"@nestjs/testing": "^7.5.1",
"@types/body-parser": "^1.19.0",
"@types/debug": "^4.1.5",
"@types/express": "^4.17.8",
"@types/jest": "^26.0.15",

View File

@ -1,4 +1,4 @@
import { Module } from '@nestjs/common';
import { MiddlewareConsumer, Module, NestModule } from '@nestjs/common';
import { ConfigModule, ConfigService } from '@nestjs/config';
import { GraphQLModule } from '@nestjs/graphql';
import { TypeOrmModule } from '@nestjs/typeorm';
@ -11,6 +11,10 @@ import { PipelinesModule } from './pipelines/pipelines.module';
import { PipelineTasksModule } from './pipeline-tasks/pipeline-tasks.module';
import configuration from './commons/config/configuration';
import { RedisModule } from 'nestjs-redis';
import { WebhooksModule } from './webhooks/webhooks.module';
import { RawBodyMiddleware } from './commons/middlewares/raw-body.middleware';
import { GiteaWebhooksController } from './webhooks/gitea-webhooks.controller';
import { ParseBodyMiddleware } from './commons/middlewares/parse-body.middleware';
@Module({
imports: [
@ -55,8 +59,17 @@ import { RedisModule } from 'nestjs-redis';
}),
inject: [ConfigService],
}),
WebhooksModule,
],
controllers: [AppController],
providers: [AppService, AppResolver],
})
export class AppModule {}
export class AppModule implements NestModule {
public configure(consumer: MiddlewareConsumer): void {
consumer
.apply(RawBodyMiddleware)
.forRoutes(GiteaWebhooksController)
.apply(ParseBodyMiddleware)
.forRoutes('*');
}
}

View File

@ -3,25 +3,49 @@ import {
Catch,
ArgumentsHost,
HttpException,
HttpStatus,
} from '@nestjs/common';
import { ApolloError } from 'apollo-server-errors';
@Catch(HttpException)
export class HttpExceptionFilter implements ExceptionFilter {
catch(exception: HttpException, host: ArgumentsHost) {
const message = exception.message;
const extensions: Record<string, any> = {};
const err = exception.getResponse();
if (typeof err === 'string') {
extensions.message = err;
} else {
Object.assign(extensions, (err as any).extension);
extensions.message = (err as any).message;
switch (host.getType<'http' | 'graphql' | string>()) {
case 'graphql': {
const message = exception.message;
const extensions: Record<string, any> = {};
const err = exception.getResponse();
if (typeof err === 'string') {
extensions.message = err;
} else {
Object.assign(extensions, (err as any).extension);
extensions.message = (err as any).message;
}
return new ApolloError(
message,
exception.getStatus().toString(),
extensions,
);
}
case 'http': {
const ctx = host.switchToHttp();
const response = ctx.getResponse();
const request = ctx.getRequest();
const status =
exception instanceof HttpException
? exception.getStatus()
: HttpStatus.INTERNAL_SERVER_ERROR;
response.status(status).json({
statusCode: status,
message: exception.message,
timestamp: new Date().toISOString(),
path: request.url,
});
}
default:
throw exception;
}
return new ApolloError(
message,
exception.getStatus().toString(),
extensions,
);
}
}

View File

@ -0,0 +1,7 @@
import { ParseBodyMiddleware } from './parse-body.middleware';
describe('ParseBodyMiddleware', () => {
it('should be defined', () => {
expect(new ParseBodyMiddleware()).toBeDefined();
});
});

View File

@ -0,0 +1,13 @@
import { Injectable, NestMiddleware } from '@nestjs/common';
import { json, urlencoded, text } from 'body-parser';
import { Request, Response, NextFunction } from 'express';
@Injectable()
export class ParseBodyMiddleware implements NestMiddleware {
use(req: Request, res: Response, next: NextFunction) {
json()(req, res, () =>
urlencoded()(req, res, () => text()(req, res, next)),
);
// next();
}
}

View File

@ -0,0 +1,7 @@
import { RawBodyMiddleware } from './raw-body.middleware';
describe('RawBodyMiddleware', () => {
it('should be defined', () => {
expect(new RawBodyMiddleware()).toBeDefined();
});
});

View File

@ -0,0 +1,10 @@
import { Injectable, NestMiddleware } from '@nestjs/common';
import { raw } from 'body-parser';
import { Request, Response, NextFunction } from 'express';
@Injectable()
export class RawBodyMiddleware implements NestMiddleware {
use(req: Request, res: Response, next: NextFunction) {
raw({ type: '*/*' })(req, res, next);
}
}

View File

@ -4,12 +4,18 @@ import { sanitize } from '@neuralegion/class-sanitizer/dist';
@Injectable()
export class SanitizePipe implements PipeTransform {
transform(value: any, metadata: ArgumentMetadata) {
// console.log(value, typeof value);
if (value instanceof Object) {
value = Object.assign(new metadata.metatype(), value);
sanitize(value);
// console.log(value);
if (
!(value instanceof Object) ||
value instanceof Buffer ||
value instanceof Array
) {
return value;
}
return value;
const constructorFunction = metadata.metatype;
if (!constructorFunction) {
return value;
}
value = Object.assign(new constructorFunction(), value);
sanitize(value);
}
}

View File

@ -6,7 +6,7 @@ import { HttpExceptionFilter } from './commons/filters/all.exception-filter';
import { SanitizePipe } from './commons/pipes/sanitize.pipe';
async function bootstrap() {
const app = await NestFactory.create(AppModule);
const app = await NestFactory.create(AppModule, { bodyParser: false });
const configService = app.get(ConfigService);
app.useGlobalPipes(new SanitizePipe());
app.useGlobalPipes(

View File

@ -10,7 +10,6 @@ import { BullModule } from '@nestjs/bull';
import { PipelineTaskConsumer } from './pipeline-task.consumer';
import {
PIPELINE_TASK_QUEUE,
PIPELINE_TASK_LOG_QUEUE,
PIPELINE_TASK_LOG_PUBSUB,
} from './pipeline-tasks.constants';
import { PipelineTaskLogsService } from './pipeline-task-logs.service';
@ -19,12 +18,9 @@ import { PubSub } from 'apollo-server-express';
@Module({
imports: [
TypeOrmModule.forFeature([PipelineTask, Pipeline]),
BullModule.registerQueue(
{
name: PIPELINE_TASK_QUEUE,
},
{ name: PIPELINE_TASK_LOG_QUEUE },
),
BullModule.registerQueue({
name: PIPELINE_TASK_QUEUE,
}),
RedisModule,
ReposModule,
],
@ -38,5 +34,6 @@ import { PubSub } from 'apollo-server-express';
useValue: new PubSub(),
},
],
exports: [PipelineTasksService],
})
export class PipelineTasksModule {}

View File

@ -32,6 +32,7 @@ describe('PipelineTasksService', () => {
({
pipelineId: 'test',
commit: 'test',
pipeline: { branch: 'master' },
units: [],
} as PipelineTask);
@ -79,6 +80,7 @@ describe('PipelineTasksService', () => {
jest
.spyOn(taskRepository, 'create')
.mockImplementation((data: any) => data);
jest.spyOn(taskRepository, 'findOne').mockImplementation(async () => null);
});
it('should be defined', () => {
@ -103,6 +105,7 @@ describe('PipelineTasksService', () => {
const save = jest
.spyOn(taskRepository, 'save')
.mockImplementation(async (data: any) => data);
const findOne = jest.spyOn(taskRepository, 'findOne');
jest
.spyOn(service, 'doNextTask')
.mockImplementation(async () => undefined);
@ -112,6 +115,7 @@ describe('PipelineTasksService', () => {
commit: 'test',
units: [],
});
expect(findOne).toBeCalled();
});
it('add task', async () => {
const lpush = jest.spyOn(redisClient, 'lpush');

View File

@ -0,0 +1,8 @@
import { IsString } from 'class-validator';
export class GiteaHookPayloadDto {
@IsString()
ref: string;
@IsString()
after: string;
}

View File

@ -0,0 +1,3 @@
export enum SourceService {
gitea = 'gitea',
}

View File

@ -0,0 +1,25 @@
import { Test, TestingModule } from '@nestjs/testing';
import { GiteaWebhooksController } from './gitea-webhooks.controller';
import { WebhooksService } from './webhooks.service';
describe('GiteaWebhooksController', () => {
let controller: GiteaWebhooksController;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
controllers: [GiteaWebhooksController],
providers: [
{
provide: WebhooksService,
useValue: {},
},
],
}).compile();
controller = module.get<GiteaWebhooksController>(GiteaWebhooksController);
});
it('should be defined', () => {
expect(controller).toBeDefined();
});
});

View File

@ -0,0 +1,40 @@
import { Body, Controller, Headers, Param, Post, Req } from '@nestjs/common';
import { validateOrReject } from 'class-validator';
import { writeFile } from 'fs/promises';
import { join } from 'path';
import { pick } from 'ramda';
import { GiteaHookPayloadDto } from './dtos/gitea-hook-payload.dto';
import { SourceService } from './enums/source-service.enum';
import { WebhookLog } from './webhook-log.entity';
import { WebhooksService } from './webhooks.service';
@Controller('gitea-webhooks')
export class GiteaWebhooksController {
constructor(private readonly service: WebhooksService) {}
@Post(':pipelineId')
async onCall(
@Headers('X-Gitea-Delivery') delivery: string,
@Headers('X-Gitea-Event') event: string,
@Headers('X-Gitea-Signature') signature: string,
@Body() body: Buffer,
@Param('pipelineId') pipelineId: string,
) {
await writeFile(join(__dirname, '../../test/data/test'), body);
const payload = Object.assign(
new GiteaHookPayloadDto(),
JSON.parse(body.toString('utf-8')),
);
await validateOrReject(payload);
await this.service.verifySignature(body, signature, 'boardcat');
return await this.service
.onCall(pipelineId, {
payload,
sourceDelivery: delivery,
sourceEvent: event,
sourceService: SourceService.gitea,
})
.then((data) =>
pick<keyof WebhookLog>(['id', 'createdAt', 'localEvent'])(data),
);
}
}

View File

@ -0,0 +1,9 @@
import { SourceService } from '../enums/source-service.enum';
import { WebhookLog } from '../webhook-log.entity';
export class CreateWebhookLogModel<T> implements Partial<WebhookLog> {
sourceDelivery: string;
sourceEvent: string;
sourceService: SourceService;
payload: T;
}

View File

@ -0,0 +1,19 @@
import { Column, Entity } from 'typeorm';
import { AppBaseEntity } from './../commons/entities/app-base-entity';
import { SourceService } from './enums/source-service.enum';
@Entity()
export class WebhookLog extends AppBaseEntity {
@Column()
sourceDelivery: string;
@Column({ type: 'enum', enum: SourceService })
sourceService: SourceService;
@Column()
sourceEvent: string;
@Column({ type: 'jsonb' })
payload: any;
@Column()
localEvent: string;
@Column({ type: 'jsonb' })
localPayload: any;
}

View File

@ -0,0 +1,20 @@
import { MiddlewareConsumer, Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { PipelineTasksModule } from '../pipeline-tasks/pipeline-tasks.module';
import { GiteaWebhooksController } from './gitea-webhooks.controller';
import { WebhookLog } from './webhook-log.entity';
import { WebhooksService } from './webhooks.service';
import { raw } from 'body-parser';
@Module({
imports: [TypeOrmModule.forFeature([WebhookLog]), PipelineTasksModule],
controllers: [GiteaWebhooksController],
providers: [WebhooksService],
})
export class WebhooksModule {
// configure(consumer: MiddlewareConsumer) {
// consumer
// .apply(raw({ type: 'application/json' }))
// .forRoutes(GiteaWebhooksController);
// }
}

View File

@ -0,0 +1,57 @@
import { UnauthorizedException } from '@nestjs/common';
import { Test, TestingModule } from '@nestjs/testing';
import { getRepositoryToken } from '@nestjs/typeorm';
import { readFile } from 'fs/promises';
import { join } from 'path';
import { Repository } from 'typeorm';
import { PipelineTasksService } from '../pipeline-tasks/pipeline-tasks.service';
import { WebhookLog } from './webhook-log.entity';
import { WebhooksService } from './webhooks.service';
describe('WebhooksService', () => {
let service: WebhooksService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
WebhooksService,
{
provide: PipelineTasksService,
useValue: {},
},
{
provide: getRepositoryToken(WebhookLog),
useValue: new Repository(),
},
],
}).compile();
service = module.get<WebhooksService>(WebhooksService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
describe('verifySignature', () => {
const signature =
'b175e07189a6106f386b62253b18b5879c4b1f3af2f11fe13a294602671e361a';
const secret = 'boardcat';
let payload: Buffer;
beforeAll(async () => {
payload = await readFile(
join(__dirname, '../../test/data/gitea-hook-payload.json.bin'),
);
});
it('must be valid', async () => {
await expect(
service.verifySignature(payload, signature, secret),
).resolves.toEqual(undefined);
});
it('must be invalid', async () => {
await expect(
service.verifySignature(payload, 'test', secret),
).rejects.toThrowError(UnauthorizedException);
});
});
});

View File

@ -0,0 +1,65 @@
import {
BadRequestException,
Injectable,
UnauthorizedException,
} from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { createHmac } from 'crypto';
import { Repository } from 'typeorm';
import { PipelineUnits } from '../pipeline-tasks/enums/pipeline-units.enum';
import { PipelineTasksService } from '../pipeline-tasks/pipeline-tasks.service';
import { GiteaHookPayloadDto } from './dtos/gitea-hook-payload.dto';
import { CreateWebhookLogModel } from './models/create-webhook-log.model';
import { WebhookLog } from './webhook-log.entity';
@Injectable()
export class WebhooksService {
constructor(
@InjectRepository(WebhookLog)
private readonly repository: Repository<WebhookLog>,
private readonly taskService: PipelineTasksService,
) {}
async onCall(
pipelineId: string,
model: CreateWebhookLogModel<GiteaHookPayloadDto>,
) {
if (model.sourceEvent.toLowerCase() === 'push') {
const taskDto = {
pipelineId,
commit: model.payload.after,
units: Object.values(PipelineUnits),
};
await this.taskService.addTask(taskDto);
return await this.repository.save(
this.repository.create({
...model,
localEvent: 'create-pipeline-task',
localPayload: taskDto,
}),
);
} else {
throw new BadRequestException('无法处理的请求');
}
}
async verifySignature(payload: any, signature: string, secret: string) {
const local = await new Promise<string>((resolve, reject) => {
const hmac = createHmac('sha256', secret);
hmac.on('readable', () => {
const data = hmac.read();
if (data) {
resolve(data.toString('hex'));
}
});
hmac.on('error', (err) => {
reject(err);
});
hmac.write(payload);
hmac.end();
});
if (local !== signature) {
throw new UnauthorizedException();
}
}
}

View File

@ -0,0 +1,115 @@
{
"secret": "boardcat",
"ref": "refs/heads/master",
"before": "429de1eaedf1da83f1e0e3ac3d8b20e771b7051c",
"after": "429de1eaedf1da83f1e0e3ac3d8b20e771b7051c",
"compare_url": "",
"commits": [
{
"id": "429de1eaedf1da83f1e0e3ac3d8b20e771b7051c",
"message": "test(pipeline-tasks): pass test cases.\n",
"url": "https://git.ivanli.cc/Fennec/fennec-be/commit/429de1eaedf1da83f1e0e3ac3d8b20e771b7051c",
"author": {
"name": "Ivan",
"email": "ivanli@live.cn",
"username": ""
},
"committer": {
"name": "Ivan",
"email": "ivanli@live.cn",
"username": ""
},
"verification": null,
"timestamp": "0001-01-01T00:00:00Z",
"added": null,
"removed": null,
"modified": null
}
],
"head_commit": null,
"repository": {
"id": 3,
"owner": {
"id": 3,
"login": "Fennec",
"full_name": "",
"email": "",
"avatar_url": "https://git.ivanli.cc/user/avatar/Fennec/-1",
"language": "",
"is_admin": false,
"last_login": "1970-01-01T08:00:00+08:00",
"created": "2021-01-30T16:46:11+08:00",
"username": "Fennec"
},
"name": "fennec-be",
"full_name": "Fennec/fennec-be",
"description": "Fennec CI/CD Back-End",
"empty": false,
"private": false,
"fork": false,
"template": false,
"parent": null,
"mirror": false,
"size": 1897,
"html_url": "https://git.ivanli.cc/Fennec/fennec-be",
"ssh_url": "ssh://gitea@git.ivanli.cc:7018/Fennec/fennec-be.git",
"clone_url": "https://git.ivanli.cc/Fennec/fennec-be.git",
"original_url": "",
"website": "",
"stars_count": 1,
"forks_count": 0,
"watchers_count": 1,
"open_issues_count": 0,
"open_pr_counter": 0,
"release_counter": 0,
"default_branch": "master",
"archived": false,
"created_at": "2021-01-31T09:58:38+08:00",
"updated_at": "2021-03-27T15:57:00+08:00",
"permissions": {
"admin": false,
"push": false,
"pull": false
},
"has_issues": true,
"internal_tracker": {
"enable_time_tracker": true,
"allow_only_contributors_to_track_time": true,
"enable_issue_dependencies": true
},
"has_wiki": true,
"has_pull_requests": true,
"has_projects": true,
"ignore_whitespace_conflicts": false,
"allow_merge_commits": true,
"allow_rebase": true,
"allow_rebase_explicit": true,
"allow_squash_merge": true,
"avatar_url": "",
"internal": false
},
"pusher": {
"id": 1,
"login": "Ivan",
"full_name": "Ivan Li",
"email": "ivan@noreply.%(DOMAIN)s",
"avatar_url": "https://git.ivanli.cc/user/avatar/Ivan/-1",
"language": "zh-CN",
"is_admin": true,
"last_login": "2021-03-26T22:28:05+08:00",
"created": "2021-01-23T18:15:30+08:00",
"username": "Ivan"
},
"sender": {
"id": 1,
"login": "Ivan",
"full_name": "Ivan Li",
"email": "ivan@noreply.%(DOMAIN)s",
"avatar_url": "https://git.ivanli.cc/user/avatar/Ivan/-1",
"language": "zh-CN",
"is_admin": true,
"last_login": "2021-03-26T22:28:05+08:00",
"created": "2021-01-23T18:15:30+08:00",
"username": "Ivan"
}
}

115
test/data/test Normal file
View File

@ -0,0 +1,115 @@
{
"secret": "boardcat",
"ref": "refs/heads/master",
"before": "429de1eaedf1da83f1e0e3ac3d8b20e771b7051c",
"after": "429de1eaedf1da83f1e0e3ac3d8b20e771b7051c",
"compare_url": "",
"commits": [
{
"id": "429de1eaedf1da83f1e0e3ac3d8b20e771b7051c",
"message": "test(pipeline-tasks): pass test cases.\n",
"url": "https://git.ivanli.cc/Fennec/fennec-be/commit/429de1eaedf1da83f1e0e3ac3d8b20e771b7051c",
"author": {
"name": "Ivan",
"email": "ivanli@live.cn",
"username": ""
},
"committer": {
"name": "Ivan",
"email": "ivanli@live.cn",
"username": ""
},
"verification": null,
"timestamp": "0001-01-01T00:00:00Z",
"added": null,
"removed": null,
"modified": null
}
],
"head_commit": null,
"repository": {
"id": 3,
"owner": {
"id": 3,
"login": "Fennec",
"full_name": "",
"email": "",
"avatar_url": "https://git.ivanli.cc/user/avatar/Fennec/-1",
"language": "",
"is_admin": false,
"last_login": "1970-01-01T08:00:00+08:00",
"created": "2021-01-30T16:46:11+08:00",
"username": "Fennec"
},
"name": "fennec-be",
"full_name": "Fennec/fennec-be",
"description": "Fennec CI/CD Back-End",
"empty": false,
"private": false,
"fork": false,
"template": false,
"parent": null,
"mirror": false,
"size": 1897,
"html_url": "https://git.ivanli.cc/Fennec/fennec-be",
"ssh_url": "ssh://gitea@git.ivanli.cc:7018/Fennec/fennec-be.git",
"clone_url": "https://git.ivanli.cc/Fennec/fennec-be.git",
"original_url": "",
"website": "",
"stars_count": 1,
"forks_count": 0,
"watchers_count": 1,
"open_issues_count": 0,
"open_pr_counter": 0,
"release_counter": 0,
"default_branch": "master",
"archived": false,
"created_at": "2021-01-31T09:58:38+08:00",
"updated_at": "2021-03-27T15:57:00+08:00",
"permissions": {
"admin": false,
"push": false,
"pull": false
},
"has_issues": true,
"internal_tracker": {
"enable_time_tracker": true,
"allow_only_contributors_to_track_time": true,
"enable_issue_dependencies": true
},
"has_wiki": true,
"has_pull_requests": true,
"has_projects": true,
"ignore_whitespace_conflicts": false,
"allow_merge_commits": true,
"allow_rebase": true,
"allow_rebase_explicit": true,
"allow_squash_merge": true,
"avatar_url": "",
"internal": false
},
"pusher": {
"id": 1,
"login": "Ivan",
"full_name": "Ivan Li",
"email": "ivan@noreply.%(DOMAIN)s",
"avatar_url": "https://git.ivanli.cc/user/avatar/Ivan/-1",
"language": "zh-CN",
"is_admin": true,
"last_login": "2021-03-26T22:28:05+08:00",
"created": "2021-01-23T18:15:30+08:00",
"username": "Ivan"
},
"sender": {
"id": 1,
"login": "Ivan",
"full_name": "Ivan Li",
"email": "ivan@noreply.%(DOMAIN)s",
"avatar_url": "https://git.ivanli.cc/user/avatar/Ivan/-1",
"language": "zh-CN",
"is_admin": true,
"last_login": "2021-03-26T22:28:05+08:00",
"created": "2021-01-23T18:15:30+08:00",
"username": "Ivan"
}
}