提供 gieea webhooks (#2)

chore: debug log 仅输出app的log

fix(commons): fix sanitize not return value.

feat(webhooks): add gitea webhooks api.

Co-authored-by: Ivan Li <ivanli@live.cn>
Co-authored-by: Ivan <ivanli@live.cn>
Reviewed-on: #2
Co-Authored-By: Ivan Li <ivan@noreply.%(DOMAIN)s>
Co-Committed-By: Ivan Li <ivan@noreply.%(DOMAIN)s>
This commit is contained in:
Ivan Li 2021-03-28 10:24:12 +08:00
parent 429de1eaed
commit da6bc9a068
22 changed files with 7935 additions and 8698 deletions

View File

@ -1,8 +1,11 @@
{ {
"cSpell.words": [ "cSpell.words": [
"Repos", "Repos",
"boardcat",
"gitea",
"lpush", "lpush",
"lrange", "lrange",
"metatype",
"rpop", "rpop",
"rpush" "rpush"
] ]

15968
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -10,8 +10,8 @@
"build": "nest build", "build": "nest build",
"format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"", "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
"start": "nest start", "start": "nest start",
"start:dev": "DEBUG=fennec:*,simple-git:* nest start --watch", "start:dev": "DEBUG=fennec:* nest start --watch",
"start:debug": "DEBUG=simple-git,simple-git:* nest start --debug --watch", "start:debug": "DEBUG=fennec:* nest start --debug --watch",
"start:prod": "node dist/main", "start:prod": "node dist/main",
"lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix", "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
"test": "jest", "test": "jest",
@ -33,6 +33,7 @@
"@types/ramda": "^0.27.38", "@types/ramda": "^0.27.38",
"apollo-server-express": "^2.19.2", "apollo-server-express": "^2.19.2",
"bcrypt": "^5.0.0", "bcrypt": "^5.0.0",
"body-parser": "^1.19.0",
"bull": "^3.20.1", "bull": "^3.20.1",
"class-transformer": "^0.3.2", "class-transformer": "^0.3.2",
"class-validator": "^0.13.1", "class-validator": "^0.13.1",
@ -51,9 +52,10 @@
"typeorm": "^0.2.30" "typeorm": "^0.2.30"
}, },
"devDependencies": { "devDependencies": {
"@nestjs/cli": "^7.5.1", "@nestjs/cli": "^7.5.7",
"@nestjs/schematics": "^7.1.3", "@nestjs/schematics": "^7.1.3",
"@nestjs/testing": "^7.5.1", "@nestjs/testing": "^7.5.1",
"@types/body-parser": "^1.19.0",
"@types/debug": "^4.1.5", "@types/debug": "^4.1.5",
"@types/express": "^4.17.8", "@types/express": "^4.17.8",
"@types/jest": "^26.0.15", "@types/jest": "^26.0.15",

View File

@ -1,4 +1,4 @@
import { Module } from '@nestjs/common'; import { MiddlewareConsumer, Module, NestModule } from '@nestjs/common';
import { ConfigModule, ConfigService } from '@nestjs/config'; import { ConfigModule, ConfigService } from '@nestjs/config';
import { GraphQLModule } from '@nestjs/graphql'; import { GraphQLModule } from '@nestjs/graphql';
import { TypeOrmModule } from '@nestjs/typeorm'; import { TypeOrmModule } from '@nestjs/typeorm';
@ -11,6 +11,10 @@ import { PipelinesModule } from './pipelines/pipelines.module';
import { PipelineTasksModule } from './pipeline-tasks/pipeline-tasks.module'; import { PipelineTasksModule } from './pipeline-tasks/pipeline-tasks.module';
import configuration from './commons/config/configuration'; import configuration from './commons/config/configuration';
import { RedisModule } from 'nestjs-redis'; import { RedisModule } from 'nestjs-redis';
import { WebhooksModule } from './webhooks/webhooks.module';
import { RawBodyMiddleware } from './commons/middlewares/raw-body.middleware';
import { GiteaWebhooksController } from './webhooks/gitea-webhooks.controller';
import { ParseBodyMiddleware } from './commons/middlewares/parse-body.middleware';
@Module({ @Module({
imports: [ imports: [
@ -55,8 +59,17 @@ import { RedisModule } from 'nestjs-redis';
}), }),
inject: [ConfigService], inject: [ConfigService],
}), }),
WebhooksModule,
], ],
controllers: [AppController], controllers: [AppController],
providers: [AppService, AppResolver], providers: [AppService, AppResolver],
}) })
export class AppModule {} export class AppModule implements NestModule {
public configure(consumer: MiddlewareConsumer): void {
consumer
.apply(RawBodyMiddleware)
.forRoutes(GiteaWebhooksController)
.apply(ParseBodyMiddleware)
.forRoutes('*');
}
}

View File

@ -3,12 +3,15 @@ import {
Catch, Catch,
ArgumentsHost, ArgumentsHost,
HttpException, HttpException,
HttpStatus,
} from '@nestjs/common'; } from '@nestjs/common';
import { ApolloError } from 'apollo-server-errors'; import { ApolloError } from 'apollo-server-errors';
@Catch(HttpException) @Catch(HttpException)
export class HttpExceptionFilter implements ExceptionFilter { export class HttpExceptionFilter implements ExceptionFilter {
catch(exception: HttpException, host: ArgumentsHost) { catch(exception: HttpException, host: ArgumentsHost) {
switch (host.getType<'http' | 'graphql' | string>()) {
case 'graphql': {
const message = exception.message; const message = exception.message;
const extensions: Record<string, any> = {}; const extensions: Record<string, any> = {};
const err = exception.getResponse(); const err = exception.getResponse();
@ -24,4 +27,25 @@ export class HttpExceptionFilter implements ExceptionFilter {
extensions, extensions,
); );
} }
case 'http': {
const ctx = host.switchToHttp();
const response = ctx.getResponse();
const request = ctx.getRequest();
const status =
exception instanceof HttpException
? exception.getStatus()
: HttpStatus.INTERNAL_SERVER_ERROR;
response.status(status).json({
statusCode: status,
message: exception.message,
timestamp: new Date().toISOString(),
path: request.url,
});
}
default:
throw exception;
}
}
} }

View File

@ -0,0 +1,7 @@
import { ParseBodyMiddleware } from './parse-body.middleware';
describe('ParseBodyMiddleware', () => {
it('should be defined', () => {
expect(new ParseBodyMiddleware()).toBeDefined();
});
});

View File

@ -0,0 +1,13 @@
import { Injectable, NestMiddleware } from '@nestjs/common';
import { json, urlencoded, text } from 'body-parser';
import { Request, Response, NextFunction } from 'express';
@Injectable()
export class ParseBodyMiddleware implements NestMiddleware {
use(req: Request, res: Response, next: NextFunction) {
json()(req, res, () =>
urlencoded()(req, res, () => text()(req, res, next)),
);
// next();
}
}

View File

@ -0,0 +1,7 @@
import { RawBodyMiddleware } from './raw-body.middleware';
describe('RawBodyMiddleware', () => {
it('should be defined', () => {
expect(new RawBodyMiddleware()).toBeDefined();
});
});

View File

@ -0,0 +1,10 @@
import { Injectable, NestMiddleware } from '@nestjs/common';
import { raw } from 'body-parser';
import { Request, Response, NextFunction } from 'express';
@Injectable()
export class RawBodyMiddleware implements NestMiddleware {
use(req: Request, res: Response, next: NextFunction) {
raw({ type: '*/*' })(req, res, next);
}
}

View File

@ -4,12 +4,24 @@ import { sanitize } from '@neuralegion/class-sanitizer/dist';
@Injectable() @Injectable()
export class SanitizePipe implements PipeTransform { export class SanitizePipe implements PipeTransform {
transform(value: any, metadata: ArgumentMetadata) { transform(value: any, metadata: ArgumentMetadata) {
// console.log(value, typeof value); if (
if (value instanceof Object) { !(value instanceof Object) ||
value = Object.assign(new metadata.metatype(), value); value instanceof Buffer ||
sanitize(value); value instanceof Array
// console.log(value); ) {
}
return value; return value;
} }
const constructorFunction = metadata.metatype;
if (!constructorFunction) {
return value;
}
value = Object.assign(new constructorFunction(), value);
try {
sanitize(value);
return value;
} catch (err) {
console.error(err);
throw err;
}
}
} }

View File

@ -6,7 +6,7 @@ import { HttpExceptionFilter } from './commons/filters/all.exception-filter';
import { SanitizePipe } from './commons/pipes/sanitize.pipe'; import { SanitizePipe } from './commons/pipes/sanitize.pipe';
async function bootstrap() { async function bootstrap() {
const app = await NestFactory.create(AppModule); const app = await NestFactory.create(AppModule, { bodyParser: false });
const configService = app.get(ConfigService); const configService = app.get(ConfigService);
app.useGlobalPipes(new SanitizePipe()); app.useGlobalPipes(new SanitizePipe());
app.useGlobalPipes( app.useGlobalPipes(

View File

@ -10,7 +10,6 @@ import { BullModule } from '@nestjs/bull';
import { PipelineTaskConsumer } from './pipeline-task.consumer'; import { PipelineTaskConsumer } from './pipeline-task.consumer';
import { import {
PIPELINE_TASK_QUEUE, PIPELINE_TASK_QUEUE,
PIPELINE_TASK_LOG_QUEUE,
PIPELINE_TASK_LOG_PUBSUB, PIPELINE_TASK_LOG_PUBSUB,
} from './pipeline-tasks.constants'; } from './pipeline-tasks.constants';
import { PipelineTaskLogsService } from './pipeline-task-logs.service'; import { PipelineTaskLogsService } from './pipeline-task-logs.service';
@ -35,5 +34,6 @@ import { PubSub } from 'apollo-server-express';
useValue: new PubSub(), useValue: new PubSub(),
}, },
], ],
exports: [PipelineTasksService],
}) })
export class PipelineTasksModule {} export class PipelineTasksModule {}

View File

@ -0,0 +1,8 @@
import { IsString } from 'class-validator';
export class GiteaHookPayloadDto {
@IsString()
ref: string;
@IsString()
after: string;
}

View File

@ -0,0 +1,3 @@
export enum SourceService {
gitea = 'gitea',
}

View File

@ -0,0 +1,25 @@
import { Test, TestingModule } from '@nestjs/testing';
import { GiteaWebhooksController } from './gitea-webhooks.controller';
import { WebhooksService } from './webhooks.service';
describe('GiteaWebhooksController', () => {
let controller: GiteaWebhooksController;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
controllers: [GiteaWebhooksController],
providers: [
{
provide: WebhooksService,
useValue: {},
},
],
}).compile();
controller = module.get<GiteaWebhooksController>(GiteaWebhooksController);
});
it('should be defined', () => {
expect(controller).toBeDefined();
});
});

View File

@ -0,0 +1,37 @@
import { Body, Controller, Headers, Param, Post } from '@nestjs/common';
import { validateOrReject } from 'class-validator';
import { pick } from 'ramda';
import { GiteaHookPayloadDto } from './dtos/gitea-hook-payload.dto';
import { SourceService } from './enums/source-service.enum';
import { WebhookLog } from './webhook-log.entity';
import { WebhooksService } from './webhooks.service';
@Controller('gitea-webhooks')
export class GiteaWebhooksController {
constructor(private readonly service: WebhooksService) {}
@Post(':pipelineId')
async onCall(
@Headers('X-Gitea-Delivery') delivery: string,
@Headers('X-Gitea-Event') event: string,
@Headers('X-Gitea-Signature') signature: string,
@Body() body: Buffer,
@Param('pipelineId') pipelineId: string,
) {
const payload = Object.assign(
new GiteaHookPayloadDto(),
JSON.parse(body.toString('utf-8')),
);
await validateOrReject(payload);
await this.service.verifySignature(body, signature, 'boardcat');
return await this.service
.onCall(pipelineId, {
payload,
sourceDelivery: delivery,
sourceEvent: event,
sourceService: SourceService.gitea,
})
.then((data) =>
pick<keyof WebhookLog>(['id', 'createdAt', 'localEvent'])(data),
);
}
}

View File

@ -0,0 +1,9 @@
import { SourceService } from '../enums/source-service.enum';
import { WebhookLog } from '../webhook-log.entity';
export class CreateWebhookLogModel<T> implements Partial<WebhookLog> {
sourceDelivery: string;
sourceEvent: string;
sourceService: SourceService;
payload: T;
}

View File

@ -0,0 +1,19 @@
import { Column, Entity } from 'typeorm';
import { AppBaseEntity } from './../commons/entities/app-base-entity';
import { SourceService } from './enums/source-service.enum';
@Entity()
export class WebhookLog extends AppBaseEntity {
@Column()
sourceDelivery: string;
@Column({ type: 'enum', enum: SourceService })
sourceService: SourceService;
@Column()
sourceEvent: string;
@Column({ type: 'jsonb' })
payload: any;
@Column()
localEvent: string;
@Column({ type: 'jsonb' })
localPayload: any;
}

View File

@ -0,0 +1,20 @@
import { MiddlewareConsumer, Module } from '@nestjs/common';
import { TypeOrmModule } from '@nestjs/typeorm';
import { PipelineTasksModule } from '../pipeline-tasks/pipeline-tasks.module';
import { GiteaWebhooksController } from './gitea-webhooks.controller';
import { WebhookLog } from './webhook-log.entity';
import { WebhooksService } from './webhooks.service';
import { raw } from 'body-parser';
@Module({
imports: [TypeOrmModule.forFeature([WebhookLog]), PipelineTasksModule],
controllers: [GiteaWebhooksController],
providers: [WebhooksService],
})
export class WebhooksModule {
// configure(consumer: MiddlewareConsumer) {
// consumer
// .apply(raw({ type: 'application/json' }))
// .forRoutes(GiteaWebhooksController);
// }
}

View File

@ -0,0 +1,57 @@
import { UnauthorizedException } from '@nestjs/common';
import { Test, TestingModule } from '@nestjs/testing';
import { getRepositoryToken } from '@nestjs/typeorm';
import { readFile } from 'fs/promises';
import { join } from 'path';
import { Repository } from 'typeorm';
import { PipelineTasksService } from '../pipeline-tasks/pipeline-tasks.service';
import { WebhookLog } from './webhook-log.entity';
import { WebhooksService } from './webhooks.service';
describe('WebhooksService', () => {
let service: WebhooksService;
beforeEach(async () => {
const module: TestingModule = await Test.createTestingModule({
providers: [
WebhooksService,
{
provide: PipelineTasksService,
useValue: {},
},
{
provide: getRepositoryToken(WebhookLog),
useValue: new Repository(),
},
],
}).compile();
service = module.get<WebhooksService>(WebhooksService);
});
it('should be defined', () => {
expect(service).toBeDefined();
});
describe('verifySignature', () => {
const signature =
'b175e07189a6106f386b62253b18b5879c4b1f3af2f11fe13a294602671e361a';
const secret = 'boardcat';
let payload: Buffer;
beforeAll(async () => {
payload = await readFile(
join(__dirname, '../../test/data/gitea-hook-payload.json.bin'),
);
});
it('must be valid', async () => {
await expect(
service.verifySignature(payload, signature, secret),
).resolves.toEqual(undefined);
});
it('must be invalid', async () => {
await expect(
service.verifySignature(payload, 'test', secret),
).rejects.toThrowError(UnauthorizedException);
});
});
});

View File

@ -0,0 +1,65 @@
import {
BadRequestException,
Injectable,
UnauthorizedException,
} from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { createHmac } from 'crypto';
import { Repository } from 'typeorm';
import { PipelineUnits } from '../pipeline-tasks/enums/pipeline-units.enum';
import { PipelineTasksService } from '../pipeline-tasks/pipeline-tasks.service';
import { GiteaHookPayloadDto } from './dtos/gitea-hook-payload.dto';
import { CreateWebhookLogModel } from './models/create-webhook-log.model';
import { WebhookLog } from './webhook-log.entity';
@Injectable()
export class WebhooksService {
constructor(
@InjectRepository(WebhookLog)
private readonly repository: Repository<WebhookLog>,
private readonly taskService: PipelineTasksService,
) {}
async onCall(
pipelineId: string,
model: CreateWebhookLogModel<GiteaHookPayloadDto>,
) {
if (model.sourceEvent.toLowerCase() === 'push') {
const taskDto = {
pipelineId,
commit: model.payload.after,
units: Object.values(PipelineUnits),
};
await this.taskService.addTask(taskDto);
return await this.repository.save(
this.repository.create({
...model,
localEvent: 'create-pipeline-task',
localPayload: taskDto,
}),
);
} else {
throw new BadRequestException('无法处理的请求');
}
}
async verifySignature(payload: any, signature: string, secret: string) {
const local = await new Promise<string>((resolve, reject) => {
const hmac = createHmac('sha256', secret);
hmac.on('readable', () => {
const data = hmac.read();
if (data) {
resolve(data.toString('hex'));
}
});
hmac.on('error', (err) => {
reject(err);
});
hmac.write(payload);
hmac.end();
});
if (local !== signature) {
throw new UnauthorizedException();
}
}
}

View File

@ -0,0 +1,115 @@
{
"secret": "boardcat",
"ref": "refs/heads/master",
"before": "429de1eaedf1da83f1e0e3ac3d8b20e771b7051c",
"after": "429de1eaedf1da83f1e0e3ac3d8b20e771b7051c",
"compare_url": "",
"commits": [
{
"id": "429de1eaedf1da83f1e0e3ac3d8b20e771b7051c",
"message": "test(pipeline-tasks): pass test cases.\n",
"url": "https://git.ivanli.cc/Fennec/fennec-be/commit/429de1eaedf1da83f1e0e3ac3d8b20e771b7051c",
"author": {
"name": "Ivan",
"email": "ivanli@live.cn",
"username": ""
},
"committer": {
"name": "Ivan",
"email": "ivanli@live.cn",
"username": ""
},
"verification": null,
"timestamp": "0001-01-01T00:00:00Z",
"added": null,
"removed": null,
"modified": null
}
],
"head_commit": null,
"repository": {
"id": 3,
"owner": {
"id": 3,
"login": "Fennec",
"full_name": "",
"email": "",
"avatar_url": "https://git.ivanli.cc/user/avatar/Fennec/-1",
"language": "",
"is_admin": false,
"last_login": "1970-01-01T08:00:00+08:00",
"created": "2021-01-30T16:46:11+08:00",
"username": "Fennec"
},
"name": "fennec-be",
"full_name": "Fennec/fennec-be",
"description": "Fennec CI/CD Back-End",
"empty": false,
"private": false,
"fork": false,
"template": false,
"parent": null,
"mirror": false,
"size": 1897,
"html_url": "https://git.ivanli.cc/Fennec/fennec-be",
"ssh_url": "ssh://gitea@git.ivanli.cc:7018/Fennec/fennec-be.git",
"clone_url": "https://git.ivanli.cc/Fennec/fennec-be.git",
"original_url": "",
"website": "",
"stars_count": 1,
"forks_count": 0,
"watchers_count": 1,
"open_issues_count": 0,
"open_pr_counter": 0,
"release_counter": 0,
"default_branch": "master",
"archived": false,
"created_at": "2021-01-31T09:58:38+08:00",
"updated_at": "2021-03-27T15:57:00+08:00",
"permissions": {
"admin": false,
"push": false,
"pull": false
},
"has_issues": true,
"internal_tracker": {
"enable_time_tracker": true,
"allow_only_contributors_to_track_time": true,
"enable_issue_dependencies": true
},
"has_wiki": true,
"has_pull_requests": true,
"has_projects": true,
"ignore_whitespace_conflicts": false,
"allow_merge_commits": true,
"allow_rebase": true,
"allow_rebase_explicit": true,
"allow_squash_merge": true,
"avatar_url": "",
"internal": false
},
"pusher": {
"id": 1,
"login": "Ivan",
"full_name": "Ivan Li",
"email": "ivan@noreply.%(DOMAIN)s",
"avatar_url": "https://git.ivanli.cc/user/avatar/Ivan/-1",
"language": "zh-CN",
"is_admin": true,
"last_login": "2021-03-26T22:28:05+08:00",
"created": "2021-01-23T18:15:30+08:00",
"username": "Ivan"
},
"sender": {
"id": 1,
"login": "Ivan",
"full_name": "Ivan Li",
"email": "ivan@noreply.%(DOMAIN)s",
"avatar_url": "https://git.ivanli.cc/user/avatar/Ivan/-1",
"language": "zh-CN",
"is_admin": true,
"last_login": "2021-03-26T22:28:05+08:00",
"created": "2021-01-23T18:15:30+08:00",
"username": "Ivan"
}
}