Compare commits
11 Commits
7e17de0f15
...
feat-jwt-a
Author | SHA1 | Date | |
---|---|---|---|
ab4ef36bf8 | |||
0a03bcd36e | |||
ec351d12f2 | |||
c3c73fbe65 | |||
02059ee54f | |||
5ed17cc04b | |||
256878890b | |||
9908bd229e | |||
07f19101a5 | |||
7d84017f9e | |||
a231a02c28 |
3
.vscode/extensions.json
vendored
Normal file
3
.vscode/extensions.json
vendored
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
{
|
||||||
|
"recommendations": []
|
||||||
|
}
|
1
.vscode/settings.json
vendored
1
.vscode/settings.json
vendored
@ -1,5 +1,6 @@
|
|||||||
{
|
{
|
||||||
"cSpell.words": [
|
"cSpell.words": [
|
||||||
|
"Mutex",
|
||||||
"Repos",
|
"Repos",
|
||||||
"amqp",
|
"amqp",
|
||||||
"boardcat",
|
"boardcat",
|
||||||
|
@ -16,5 +16,9 @@ db:
|
|||||||
prefix: fennec
|
prefix: fennec
|
||||||
rabbitmq:
|
rabbitmq:
|
||||||
uri: 'amqp://fennec:fennec@192.168.31.194:5672'
|
uri: 'amqp://fennec:fennec@192.168.31.194:5672'
|
||||||
|
etcd:
|
||||||
|
hosts:
|
||||||
|
- 'http://192.168.31.194:2379'
|
||||||
|
|
||||||
workspaces:
|
workspaces:
|
||||||
root: '/Users/ivanli/Projects/fennec/workspaces'
|
root: '/Users/ivanli/Projects/fennec/workspaces'
|
14
ecosystem.config.js
Normal file
14
ecosystem.config.js
Normal file
@ -0,0 +1,14 @@
|
|||||||
|
module.exports = {
|
||||||
|
apps: [
|
||||||
|
{
|
||||||
|
name: 'fennec-be',
|
||||||
|
script: 'npm',
|
||||||
|
args: 'run start:prod',
|
||||||
|
watch: false,
|
||||||
|
ignore_watch: ['node_modules'],
|
||||||
|
log_date_format: 'MM-DD HH:mm:ss.SSS Z',
|
||||||
|
env: {},
|
||||||
|
max_restarts: 5,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
};
|
12508
package-lock.json
generated
12508
package-lock.json
generated
File diff suppressed because it is too large
Load Diff
15
package.json
15
package.json
@ -1,8 +1,8 @@
|
|||||||
{
|
{
|
||||||
"name": "fennec-be",
|
"name": "fennec-be",
|
||||||
"version": "0.0.1",
|
"version": "0.1.1",
|
||||||
"description": "",
|
"description": "a ci/cd project.",
|
||||||
"author": "",
|
"author": "Ivan Li\b<ivanli2048@gmail.com>",
|
||||||
"private": true,
|
"private": true,
|
||||||
"license": "UNLICENSED",
|
"license": "UNLICENSED",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
@ -22,7 +22,7 @@
|
|||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@golevelup/nestjs-rabbitmq": "^1.16.1",
|
"@golevelup/nestjs-rabbitmq": "^1.16.1",
|
||||||
"@nestjs/bull": "^0.3.1",
|
"@nestjs-lib/auth": "^0.2.0",
|
||||||
"@nestjs/common": "^7.5.1",
|
"@nestjs/common": "^7.5.1",
|
||||||
"@nestjs/config": "^0.6.2",
|
"@nestjs/config": "^0.6.2",
|
||||||
"@nestjs/core": "^7.5.1",
|
"@nestjs/core": "^7.5.1",
|
||||||
@ -30,19 +30,19 @@
|
|||||||
"@nestjs/platform-express": "^7.5.1",
|
"@nestjs/platform-express": "^7.5.1",
|
||||||
"@nestjs/typeorm": "^7.1.5",
|
"@nestjs/typeorm": "^7.1.5",
|
||||||
"@types/amqplib": "^0.8.0",
|
"@types/amqplib": "^0.8.0",
|
||||||
"@types/bull": "^3.15.0",
|
|
||||||
"@types/ramda": "^0.27.38",
|
"@types/ramda": "^0.27.38",
|
||||||
"apollo-server-express": "^2.19.2",
|
"apollo-server-express": "^2.19.2",
|
||||||
"bcrypt": "^5.0.0",
|
"bcrypt": "^5.0.0",
|
||||||
"body-parser": "^1.19.0",
|
"body-parser": "^1.19.0",
|
||||||
"bull": "^3.20.1",
|
|
||||||
"class-transformer": "^0.3.2",
|
"class-transformer": "^0.3.2",
|
||||||
"class-validator": "^0.13.1",
|
"class-validator": "^0.13.1",
|
||||||
"debug": "^4.3.1",
|
"debug": "^4.3.1",
|
||||||
"graphql": "^15.5.0",
|
"graphql": "^15.5.0",
|
||||||
"graphql-tools": "^7.0.2",
|
"graphql-tools": "^7.0.2",
|
||||||
"ioredis": "^4.25.0",
|
"ioredis": "^4.25.0",
|
||||||
|
"jose": "^3.14.0",
|
||||||
"js-yaml": "^4.0.0",
|
"js-yaml": "^4.0.0",
|
||||||
|
"nestjs-etcd": "^0.2.0",
|
||||||
"nestjs-pino": "^1.4.0",
|
"nestjs-pino": "^1.4.0",
|
||||||
"nestjs-redis": "^1.2.8",
|
"nestjs-redis": "^1.2.8",
|
||||||
"observable-to-async-generator": "^1.0.1-rc",
|
"observable-to-async-generator": "^1.0.1-rc",
|
||||||
@ -96,6 +96,9 @@
|
|||||||
"collectCoverageFrom": [
|
"collectCoverageFrom": [
|
||||||
"**/*.(t|j)s"
|
"**/*.(t|j)s"
|
||||||
],
|
],
|
||||||
|
"moduleNameMapper": {
|
||||||
|
"^jose/(.*)$": "<rootDir>/../node_modules/jose/dist/node/cjs/$1"
|
||||||
|
},
|
||||||
"coverageDirectory": "../coverage",
|
"coverageDirectory": "../coverage",
|
||||||
"testEnvironment": "node"
|
"testEnvironment": "node"
|
||||||
}
|
}
|
||||||
|
@ -1,3 +1,4 @@
|
|||||||
|
import { CommonsModule } from './commons/commons.module';
|
||||||
import { MiddlewareConsumer, Module, NestModule } from '@nestjs/common';
|
import { MiddlewareConsumer, Module, NestModule } from '@nestjs/common';
|
||||||
import { ConfigModule, ConfigService } from '@nestjs/config';
|
import { ConfigModule, ConfigService } from '@nestjs/config';
|
||||||
import { GraphQLModule } from '@nestjs/graphql';
|
import { GraphQLModule } from '@nestjs/graphql';
|
||||||
@ -12,13 +13,13 @@ import { PipelineTasksModule } from './pipeline-tasks/pipeline-tasks.module';
|
|||||||
import configuration from './commons/config/configuration';
|
import configuration from './commons/config/configuration';
|
||||||
import { RedisModule } from 'nestjs-redis';
|
import { RedisModule } from 'nestjs-redis';
|
||||||
import { WebhooksModule } from './webhooks/webhooks.module';
|
import { WebhooksModule } from './webhooks/webhooks.module';
|
||||||
import { RawBodyMiddleware } from './commons/middlewares/raw-body.middleware';
|
import { RawBodyMiddleware } from './commons/middleware/raw-body.middleware';
|
||||||
import { GiteaWebhooksController } from './webhooks/gitea-webhooks.controller';
|
import { GiteaWebhooksController } from './webhooks/gitea-webhooks.controller';
|
||||||
import { ParseBodyMiddleware } from './commons/middlewares/parse-body.middleware';
|
import { ParseBodyMiddleware } from './commons/middleware/parse-body.middleware';
|
||||||
import { BullModule } from '@nestjs/bull';
|
|
||||||
import { LoggerModule } from 'nestjs-pino';
|
import { LoggerModule } from 'nestjs-pino';
|
||||||
|
import { EtcdModule } from 'nestjs-etcd';
|
||||||
import pinoPretty from 'pino-pretty';
|
import pinoPretty from 'pino-pretty';
|
||||||
|
import { fromPairs, map, pipe, toPairs } from 'ramda';
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
imports: [
|
imports: [
|
||||||
@ -63,16 +64,21 @@ import pinoPretty from 'pino-pretty';
|
|||||||
playground: true,
|
playground: true,
|
||||||
autoSchemaFile: true,
|
autoSchemaFile: true,
|
||||||
installSubscriptionHandlers: true,
|
installSubscriptionHandlers: true,
|
||||||
}),
|
context: ({ req, connection, ...args }) => {
|
||||||
inject: [ConfigService],
|
return connection ? { req: connection.context } : { req };
|
||||||
}),
|
},
|
||||||
BullModule.forRootAsync({
|
subscriptions: {
|
||||||
imports: [ConfigModule],
|
onConnect: (connectionParams: Record<string, string>) => {
|
||||||
useFactory: (configService: ConfigService) => ({
|
const connectionParamsWithLowerKeys = pipe(
|
||||||
redis: {
|
toPairs,
|
||||||
host: configService.get<string>('db.redis.host', 'localhost'),
|
map(([key, value]) => [key.toLowerCase(), value]),
|
||||||
port: configService.get<number>('db.redis.port', undefined),
|
fromPairs,
|
||||||
password: configService.get<string>('db.redis.password', undefined),
|
)(connectionParams);
|
||||||
|
|
||||||
|
return {
|
||||||
|
headers: connectionParamsWithLowerKeys,
|
||||||
|
};
|
||||||
|
},
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
inject: [ConfigService],
|
inject: [ConfigService],
|
||||||
@ -91,7 +97,15 @@ import pinoPretty from 'pino-pretty';
|
|||||||
}),
|
}),
|
||||||
inject: [ConfigService],
|
inject: [ConfigService],
|
||||||
}),
|
}),
|
||||||
|
EtcdModule.forRootAsync({
|
||||||
|
imports: [ConfigModule],
|
||||||
|
useFactory: (configService: ConfigService) => ({
|
||||||
|
hosts: configService.get<string>('db.etcd.hosts', 'localhost:2379'),
|
||||||
|
}),
|
||||||
|
inject: [ConfigService],
|
||||||
|
}),
|
||||||
WebhooksModule,
|
WebhooksModule,
|
||||||
|
CommonsModule,
|
||||||
],
|
],
|
||||||
controllers: [AppController],
|
controllers: [AppController],
|
||||||
providers: [AppService, AppResolver],
|
providers: [AppService, AppResolver],
|
||||||
|
@ -1,8 +1,11 @@
|
|||||||
import { Module } from '@nestjs/common';
|
import { Module } from '@nestjs/common';
|
||||||
import { PasswordConverter } from './services/password-converter';
|
import { PasswordConverter } from './services/password-converter';
|
||||||
|
import { RedisMutexModule } from './redis-mutex/redis-mutex.module';
|
||||||
|
import { AuthModule } from '@nestjs-lib/auth';
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
|
imports: [RedisMutexModule, AuthModule],
|
||||||
providers: [PasswordConverter],
|
providers: [PasswordConverter],
|
||||||
exports: [PasswordConverter],
|
exports: [PasswordConverter, RedisMutexModule, AuthModule],
|
||||||
})
|
})
|
||||||
export class CommonsModule {}
|
export class CommonsModule {}
|
||||||
|
@ -6,13 +6,18 @@ import {
|
|||||||
HttpStatus,
|
HttpStatus,
|
||||||
} from '@nestjs/common';
|
} from '@nestjs/common';
|
||||||
import { ApolloError } from 'apollo-server-errors';
|
import { ApolloError } from 'apollo-server-errors';
|
||||||
|
import { PinoLogger, InjectPinoLogger } from 'nestjs-pino';
|
||||||
|
|
||||||
@Catch(HttpException)
|
@Catch(HttpException)
|
||||||
export class HttpExceptionFilter implements ExceptionFilter {
|
export class HttpExceptionFilter implements ExceptionFilter {
|
||||||
|
constructor(
|
||||||
|
@InjectPinoLogger(HttpExceptionFilter.name)
|
||||||
|
private readonly logger: PinoLogger,
|
||||||
|
) {}
|
||||||
catch(exception: HttpException, host: ArgumentsHost) {
|
catch(exception: HttpException, host: ArgumentsHost) {
|
||||||
switch (host.getType<'http' | 'graphql' | string>()) {
|
switch (host.getType<'http' | 'graphql' | string>()) {
|
||||||
case 'graphql': {
|
case 'graphql': {
|
||||||
const message = exception.message;
|
const errorName = exception.message;
|
||||||
const extensions: Record<string, any> = {};
|
const extensions: Record<string, any> = {};
|
||||||
const err = exception.getResponse();
|
const err = exception.getResponse();
|
||||||
if (typeof err === 'string') {
|
if (typeof err === 'string') {
|
||||||
@ -21,8 +26,10 @@ export class HttpExceptionFilter implements ExceptionFilter {
|
|||||||
Object.assign(extensions, (err as any).extension);
|
Object.assign(extensions, (err as any).extension);
|
||||||
extensions.message = (err as any).message;
|
extensions.message = (err as any).message;
|
||||||
}
|
}
|
||||||
|
extensions.error = errorName;
|
||||||
|
this.logger.error(extensions);
|
||||||
return new ApolloError(
|
return new ApolloError(
|
||||||
message,
|
extensions.message,
|
||||||
exception.getStatus().toString(),
|
exception.getStatus().toString(),
|
||||||
extensions,
|
extensions,
|
||||||
);
|
);
|
||||||
|
10
src/commons/redis-mutex/redis-mutex.module.ts
Normal file
10
src/commons/redis-mutex/redis-mutex.module.ts
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
import { Module } from '@nestjs/common';
|
||||||
|
import { RedisMutexService } from './redis-mutex.service';
|
||||||
|
import { RedisModule } from 'nestjs-redis';
|
||||||
|
|
||||||
|
@Module({
|
||||||
|
imports: [RedisModule],
|
||||||
|
providers: [RedisMutexService],
|
||||||
|
exports: [RedisMutexService],
|
||||||
|
})
|
||||||
|
export class RedisMutexModule {}
|
25
src/commons/redis-mutex/redis-mutex.service.spec.ts
Normal file
25
src/commons/redis-mutex/redis-mutex.service.spec.ts
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
import { Test, TestingModule } from '@nestjs/testing';
|
||||||
|
import { RedisService } from 'nestjs-redis';
|
||||||
|
import { RedisMutexService } from './redis-mutex.service';
|
||||||
|
|
||||||
|
describe('RedisMutexService', () => {
|
||||||
|
let service: RedisMutexService;
|
||||||
|
|
||||||
|
beforeEach(async () => {
|
||||||
|
const module: TestingModule = await Test.createTestingModule({
|
||||||
|
providers: [
|
||||||
|
RedisMutexService,
|
||||||
|
{
|
||||||
|
provide: RedisService,
|
||||||
|
useValue: {},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}).compile();
|
||||||
|
|
||||||
|
service = module.get<RedisMutexService>(RedisMutexService);
|
||||||
|
});
|
||||||
|
|
||||||
|
it('should be defined', () => {
|
||||||
|
expect(service).toBeDefined();
|
||||||
|
});
|
||||||
|
});
|
71
src/commons/redis-mutex/redis-mutex.service.ts
Normal file
71
src/commons/redis-mutex/redis-mutex.service.ts
Normal file
@ -0,0 +1,71 @@
|
|||||||
|
import { Injectable } from '@nestjs/common';
|
||||||
|
import { RedisService } from 'nestjs-redis';
|
||||||
|
import * as uuid from 'uuid';
|
||||||
|
import { ApplicationException } from '../exceptions/application.exception';
|
||||||
|
|
||||||
|
export interface RedisMutexOption {
|
||||||
|
/**
|
||||||
|
* seconds
|
||||||
|
*/
|
||||||
|
expires?: number;
|
||||||
|
/**
|
||||||
|
* seconds
|
||||||
|
*/
|
||||||
|
timeout?: number | null;
|
||||||
|
/**
|
||||||
|
* milliseconds
|
||||||
|
*/
|
||||||
|
retryDelay?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class RedisMutexService {
|
||||||
|
constructor(private readonly redisClient: RedisService) {}
|
||||||
|
|
||||||
|
public async lock(
|
||||||
|
key: string,
|
||||||
|
{ expires = 100, timeout = 10, retryDelay = 100 }: RedisMutexOption = {
|
||||||
|
expires: 100,
|
||||||
|
timeout: 10,
|
||||||
|
retryDelay: 100,
|
||||||
|
},
|
||||||
|
) {
|
||||||
|
const redisKey = `${'mutex-lock'}:${key}`;
|
||||||
|
const redis = this.redisClient.getClient();
|
||||||
|
const value = uuid.v4();
|
||||||
|
const timeoutAt = timeout ? Date.now() + timeout * 1000 : null;
|
||||||
|
|
||||||
|
while (
|
||||||
|
!(await redis
|
||||||
|
.set(redisKey, value, 'EX', expires, 'NX')
|
||||||
|
.then(() => true)
|
||||||
|
.catch(() => false))
|
||||||
|
) {
|
||||||
|
if (timeoutAt && timeoutAt > Date.now()) {
|
||||||
|
throw new ApplicationException('lock timeout');
|
||||||
|
}
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, retryDelay));
|
||||||
|
}
|
||||||
|
|
||||||
|
const renewTimer = setInterval(() => {
|
||||||
|
redis.expire(redisKey, expires);
|
||||||
|
}, (expires * 1000) / 2);
|
||||||
|
|
||||||
|
return async () => {
|
||||||
|
clearInterval(renewTimer);
|
||||||
|
await redis.eval(
|
||||||
|
`
|
||||||
|
if redis.call("get", KEYS[1]) == ARGV[1]
|
||||||
|
then
|
||||||
|
return redis.call("del", KEYS[1])
|
||||||
|
else
|
||||||
|
return 0
|
||||||
|
end
|
||||||
|
`,
|
||||||
|
1,
|
||||||
|
redisKey,
|
||||||
|
value,
|
||||||
|
);
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
@ -1,3 +1,4 @@
|
|||||||
|
import { PinoLogger } from 'nestjs-pino';
|
||||||
import { ValidationPipe } from '@nestjs/common';
|
import { ValidationPipe } from '@nestjs/common';
|
||||||
import { ConfigService } from '@nestjs/config';
|
import { ConfigService } from '@nestjs/config';
|
||||||
import { NestFactory } from '@nestjs/core';
|
import { NestFactory } from '@nestjs/core';
|
||||||
@ -14,7 +15,8 @@ async function bootstrap() {
|
|||||||
transform: true,
|
transform: true,
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
app.useGlobalFilters(new HttpExceptionFilter());
|
const httpExceptionFilterLogger = await app.resolve(PinoLogger);
|
||||||
|
app.useGlobalFilters(new HttpExceptionFilter(httpExceptionFilterLogger));
|
||||||
await app.listen(configService.get<number>('http.port'));
|
await app.listen(configService.get<number>('http.port'));
|
||||||
}
|
}
|
||||||
bootstrap();
|
bootstrap();
|
||||||
|
@ -1,4 +1,6 @@
|
|||||||
import { Field, InputType, Int, ObjectType } from '@nestjs/graphql';
|
import { Field, InputType, Int, ObjectType } from '@nestjs/graphql';
|
||||||
|
import { Type } from 'class-transformer';
|
||||||
|
import { IsInstance, isInstance, ValidateNested } from 'class-validator';
|
||||||
import { WorkUnit } from './work-unit.model';
|
import { WorkUnit } from './work-unit.model';
|
||||||
|
|
||||||
@InputType('WorkUnitMetadataInput')
|
@InputType('WorkUnitMetadataInput')
|
||||||
@ -6,5 +8,9 @@ import { WorkUnit } from './work-unit.model';
|
|||||||
export class WorkUnitMetadata {
|
export class WorkUnitMetadata {
|
||||||
@Field(() => Int)
|
@Field(() => Int)
|
||||||
version = 1;
|
version = 1;
|
||||||
|
|
||||||
|
@Type(() => WorkUnit)
|
||||||
|
@IsInstance(WorkUnit, { each: true })
|
||||||
|
@ValidateNested({ each: true })
|
||||||
units: WorkUnit[];
|
units: WorkUnit[];
|
||||||
}
|
}
|
||||||
|
@ -1,4 +1,5 @@
|
|||||||
import { Field, InputType, ObjectType } from '@nestjs/graphql';
|
import { Field, InputType, ObjectType } from '@nestjs/graphql';
|
||||||
|
import { IsNotEmpty } from 'class-validator';
|
||||||
import {
|
import {
|
||||||
PipelineUnits,
|
PipelineUnits,
|
||||||
PipelineUnits as PipelineUnitTypes,
|
PipelineUnits as PipelineUnitTypes,
|
||||||
@ -9,5 +10,7 @@ import {
|
|||||||
export class WorkUnit {
|
export class WorkUnit {
|
||||||
@Field(() => PipelineUnits)
|
@Field(() => PipelineUnits)
|
||||||
type: PipelineUnitTypes;
|
type: PipelineUnitTypes;
|
||||||
|
|
||||||
|
@IsNotEmpty({ each: true })
|
||||||
scripts: string[];
|
scripts: string[];
|
||||||
}
|
}
|
||||||
|
@ -16,13 +16,14 @@ import {
|
|||||||
} from './pipeline-tasks.constants';
|
} from './pipeline-tasks.constants';
|
||||||
import { PipelineTaskLogger } from './pipeline-task.logger';
|
import { PipelineTaskLogger } from './pipeline-task.logger';
|
||||||
import { PipelineTaskFlushService } from './pipeline-task-flush.service';
|
import { PipelineTaskFlushService } from './pipeline-task-flush.service';
|
||||||
|
import { CommonsModule } from '../commons/commons.module';
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
imports: [
|
imports: [
|
||||||
|
CommonsModule,
|
||||||
TypeOrmModule.forFeature([PipelineTask, Pipeline]),
|
TypeOrmModule.forFeature([PipelineTask, Pipeline]),
|
||||||
RedisModule,
|
RedisModule,
|
||||||
ReposModule,
|
ReposModule,
|
||||||
|
|
||||||
RabbitMQModule.forRootAsync(RabbitMQModule, {
|
RabbitMQModule.forRootAsync(RabbitMQModule, {
|
||||||
imports: [ConfigModule],
|
imports: [ConfigModule],
|
||||||
useFactory: (configService: ConfigService) => ({
|
useFactory: (configService: ConfigService) => ({
|
||||||
@ -36,22 +37,6 @@ import { PipelineTaskFlushService } from './pipeline-task-flush.service';
|
|||||||
autoDelete: true,
|
autoDelete: true,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
{
|
|
||||||
name: 'stop-pipeline-task',
|
|
||||||
type: 'fanout',
|
|
||||||
options: {
|
|
||||||
durable: true,
|
|
||||||
autoDelete: true,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
|
||||||
name: 'update-pipeline-task',
|
|
||||||
type: 'fanout',
|
|
||||||
options: {
|
|
||||||
durable: false,
|
|
||||||
autoDelete: true,
|
|
||||||
},
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
name: EXCHANGE_PIPELINE_TASK_FANOUT,
|
name: EXCHANGE_PIPELINE_TASK_FANOUT,
|
||||||
type: 'fanout',
|
type: 'fanout',
|
||||||
|
@ -7,7 +7,9 @@ import { plainToClass } from 'class-transformer';
|
|||||||
import { PipelineTaskLogger } from './pipeline-task.logger';
|
import { PipelineTaskLogger } from './pipeline-task.logger';
|
||||||
import { observableToAsyncIterable } from '@graphql-tools/utils';
|
import { observableToAsyncIterable } from '@graphql-tools/utils';
|
||||||
import { PipelineTaskEvent } from './models/pipeline-task-event';
|
import { PipelineTaskEvent } from './models/pipeline-task-event';
|
||||||
|
import { Roles, AccountRole } from '@nestjs-lib/auth';
|
||||||
|
|
||||||
|
@Roles(AccountRole.admin, AccountRole.super)
|
||||||
@Resolver()
|
@Resolver()
|
||||||
export class PipelineTasksResolver {
|
export class PipelineTasksResolver {
|
||||||
constructor(
|
constructor(
|
||||||
|
@ -1,3 +1,4 @@
|
|||||||
|
import { Roles, AccountRole } from '@nestjs-lib/auth';
|
||||||
import { Query } from '@nestjs/graphql';
|
import { Query } from '@nestjs/graphql';
|
||||||
import {
|
import {
|
||||||
Args,
|
Args,
|
||||||
@ -10,6 +11,7 @@ import { PipelineTasksService } from '../pipeline-tasks/pipeline-tasks.service';
|
|||||||
import { Commit, LogFields } from '../repos/dtos/log-list.model';
|
import { Commit, LogFields } from '../repos/dtos/log-list.model';
|
||||||
import { PipelinesService } from './pipelines.service';
|
import { PipelinesService } from './pipelines.service';
|
||||||
|
|
||||||
|
@Roles(AccountRole.admin, AccountRole.super)
|
||||||
@Resolver(() => Commit)
|
@Resolver(() => Commit)
|
||||||
export class CommitLogsResolver {
|
export class CommitLogsResolver {
|
||||||
constructor(
|
constructor(
|
||||||
|
@ -1,11 +1,13 @@
|
|||||||
|
import { Type } from 'class-transformer';
|
||||||
import { InputType } from '@nestjs/graphql';
|
import { InputType } from '@nestjs/graphql';
|
||||||
import { WorkUnitMetadata } from '../../pipeline-tasks/models/work-unit-metadata.model';
|
import { WorkUnitMetadata } from '../../pipeline-tasks/models/work-unit-metadata.model';
|
||||||
import {
|
import {
|
||||||
IsObject,
|
IsInstance,
|
||||||
IsOptional,
|
IsOptional,
|
||||||
IsString,
|
IsString,
|
||||||
IsUUID,
|
IsUUID,
|
||||||
MaxLength,
|
MaxLength,
|
||||||
|
ValidateNested,
|
||||||
} from 'class-validator';
|
} from 'class-validator';
|
||||||
|
|
||||||
@InputType({ isAbstract: true })
|
@InputType({ isAbstract: true })
|
||||||
@ -21,7 +23,9 @@ export class CreatePipelineInput {
|
|||||||
@MaxLength(32)
|
@MaxLength(32)
|
||||||
name: string;
|
name: string;
|
||||||
|
|
||||||
|
@Type(() => WorkUnitMetadata)
|
||||||
@IsOptional()
|
@IsOptional()
|
||||||
@IsObject()
|
@ValidateNested()
|
||||||
|
@IsInstance(WorkUnitMetadata)
|
||||||
workUnitMetadata: WorkUnitMetadata;
|
workUnitMetadata: WorkUnitMetadata;
|
||||||
}
|
}
|
||||||
|
@ -8,9 +8,11 @@ import { PipelineTasksModule } from '../pipeline-tasks/pipeline-tasks.module';
|
|||||||
import { ReposModule } from '../repos/repos.module';
|
import { ReposModule } from '../repos/repos.module';
|
||||||
import { RabbitMQModule } from '@golevelup/nestjs-rabbitmq';
|
import { RabbitMQModule } from '@golevelup/nestjs-rabbitmq';
|
||||||
import { ConfigModule, ConfigService } from '@nestjs/config';
|
import { ConfigModule, ConfigService } from '@nestjs/config';
|
||||||
|
import { CommonsModule } from '../commons/commons.module';
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
imports: [
|
imports: [
|
||||||
|
CommonsModule,
|
||||||
TypeOrmModule.forFeature([Pipeline]),
|
TypeOrmModule.forFeature([Pipeline]),
|
||||||
PipelineTasksModule,
|
PipelineTasksModule,
|
||||||
RabbitMQModule.forRootAsync(RabbitMQModule, {
|
RabbitMQModule.forRootAsync(RabbitMQModule, {
|
||||||
|
@ -4,7 +4,9 @@ import { UpdatePipelineInput } from './dtos/update-pipeline.input';
|
|||||||
import { Pipeline } from './pipeline.entity';
|
import { Pipeline } from './pipeline.entity';
|
||||||
import { PipelinesService } from './pipelines.service';
|
import { PipelinesService } from './pipelines.service';
|
||||||
import { ListPipelineArgs } from './dtos/list-pipelines.args';
|
import { ListPipelineArgs } from './dtos/list-pipelines.args';
|
||||||
|
import { Roles, AccountRole } from '@nestjs-lib/auth';
|
||||||
|
|
||||||
|
@Roles(AccountRole.admin, AccountRole.super)
|
||||||
@Resolver()
|
@Resolver()
|
||||||
export class PipelinesResolver {
|
export class PipelinesResolver {
|
||||||
constructor(private readonly service: PipelinesService) {}
|
constructor(private readonly service: PipelinesService) {}
|
||||||
|
@ -57,7 +57,7 @@ export class PipelinesService extends BaseDbService<Pipeline> {
|
|||||||
exchange: EXCHANGE_REPO,
|
exchange: EXCHANGE_REPO,
|
||||||
routingKey: getAppInstanceRouteKey(ROUTE_FETCH, appInstance),
|
routingKey: getAppInstanceRouteKey(ROUTE_FETCH, appInstance),
|
||||||
payload: pipeline,
|
payload: pipeline,
|
||||||
timeout: 30_000,
|
timeout: 120_000,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
async listCommits(pipeline: Pipeline) {
|
async listCommits(pipeline: Pipeline) {
|
||||||
|
3
src/projects/projects.constants.ts
Normal file
3
src/projects/projects.constants.ts
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
export const EXCHANGE_PROJECT_TOPIC = 'project.topic';
|
||||||
|
export const EXCHANGE_PROJECT_FANOUT = 'project.fanout';
|
||||||
|
export const ROUTE_PROJECT_CHANGE = 'project-change';
|
@ -3,9 +3,33 @@ import { ProjectsService } from './projects.service';
|
|||||||
import { ProjectsResolver } from './projects.resolver';
|
import { ProjectsResolver } from './projects.resolver';
|
||||||
import { TypeOrmModule } from '@nestjs/typeorm';
|
import { TypeOrmModule } from '@nestjs/typeorm';
|
||||||
import { Project } from './project.entity';
|
import { Project } from './project.entity';
|
||||||
|
import { RabbitMQModule } from '@golevelup/nestjs-rabbitmq';
|
||||||
|
import { ConfigModule, ConfigService } from '@nestjs/config';
|
||||||
|
import { EXCHANGE_PROJECT_FANOUT } from './projects.constants';
|
||||||
|
import { CommonsModule } from '../commons/commons.module';
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
imports: [TypeOrmModule.forFeature([Project])],
|
imports: [
|
||||||
|
CommonsModule,
|
||||||
|
TypeOrmModule.forFeature([Project]),
|
||||||
|
RabbitMQModule.forRootAsync(RabbitMQModule, {
|
||||||
|
imports: [ConfigModule],
|
||||||
|
useFactory: (configService: ConfigService) => ({
|
||||||
|
uri: configService.get<string>('db.rabbitmq.uri'),
|
||||||
|
exchanges: [
|
||||||
|
{
|
||||||
|
name: EXCHANGE_PROJECT_FANOUT,
|
||||||
|
type: 'fanout',
|
||||||
|
options: {
|
||||||
|
durable: false,
|
||||||
|
autoDelete: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
inject: [ConfigService],
|
||||||
|
}),
|
||||||
|
],
|
||||||
providers: [ProjectsService, ProjectsResolver],
|
providers: [ProjectsService, ProjectsResolver],
|
||||||
exports: [ProjectsService],
|
exports: [ProjectsService],
|
||||||
})
|
})
|
||||||
|
@ -1,9 +1,11 @@
|
|||||||
|
import { AccountRole, Roles } from '@nestjs-lib/auth';
|
||||||
import { Args, Mutation, Query, Resolver } from '@nestjs/graphql';
|
import { Args, Mutation, Query, Resolver } from '@nestjs/graphql';
|
||||||
import { CreateProjectInput } from './dtos/create-project.input';
|
import { CreateProjectInput } from './dtos/create-project.input';
|
||||||
import { UpdateProjectInput } from './dtos/update-project.input';
|
import { UpdateProjectInput } from './dtos/update-project.input';
|
||||||
import { Project } from './project.entity';
|
import { Project } from './project.entity';
|
||||||
import { ProjectsService } from './projects.service';
|
import { ProjectsService } from './projects.service';
|
||||||
|
|
||||||
|
@Roles(AccountRole.admin, AccountRole.super)
|
||||||
@Resolver(() => Project)
|
@Resolver(() => Project)
|
||||||
export class ProjectsResolver {
|
export class ProjectsResolver {
|
||||||
constructor(private readonly service: ProjectsService) {}
|
constructor(private readonly service: ProjectsService) {}
|
||||||
@ -20,7 +22,7 @@ export class ProjectsResolver {
|
|||||||
@Mutation(() => Project)
|
@Mutation(() => Project)
|
||||||
async createProject(
|
async createProject(
|
||||||
@Args('project', { type: () => CreateProjectInput })
|
@Args('project', { type: () => CreateProjectInput })
|
||||||
dto: UpdateProjectInput,
|
dto: CreateProjectInput,
|
||||||
) {
|
) {
|
||||||
return await this.service.create(dto);
|
return await this.service.create(dto);
|
||||||
}
|
}
|
||||||
|
@ -2,6 +2,7 @@ import { Test, TestingModule } from '@nestjs/testing';
|
|||||||
import { ProjectsService } from './projects.service';
|
import { ProjectsService } from './projects.service';
|
||||||
import { getRepositoryToken } from '@nestjs/typeorm';
|
import { getRepositoryToken } from '@nestjs/typeorm';
|
||||||
import { Project } from './project.entity';
|
import { Project } from './project.entity';
|
||||||
|
import { AmqpConnection } from '@golevelup/nestjs-rabbitmq';
|
||||||
|
|
||||||
describe('ProjectsService', () => {
|
describe('ProjectsService', () => {
|
||||||
let service: ProjectsService;
|
let service: ProjectsService;
|
||||||
@ -14,6 +15,10 @@ describe('ProjectsService', () => {
|
|||||||
provide: getRepositoryToken(Project),
|
provide: getRepositoryToken(Project),
|
||||||
useValue: {},
|
useValue: {},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
provide: AmqpConnection,
|
||||||
|
useValue: {},
|
||||||
|
},
|
||||||
],
|
],
|
||||||
}).compile();
|
}).compile();
|
||||||
|
|
||||||
|
@ -5,6 +5,11 @@ import { Repository } from 'typeorm';
|
|||||||
import { CreateProjectInput } from './dtos/create-project.input';
|
import { CreateProjectInput } from './dtos/create-project.input';
|
||||||
import { Project } from './project.entity';
|
import { Project } from './project.entity';
|
||||||
import { UpdateProjectInput } from './dtos/update-project.input';
|
import { UpdateProjectInput } from './dtos/update-project.input';
|
||||||
|
import { AmqpConnection } from '@golevelup/nestjs-rabbitmq';
|
||||||
|
import {
|
||||||
|
EXCHANGE_PROJECT_FANOUT,
|
||||||
|
ROUTE_PROJECT_CHANGE,
|
||||||
|
} from './projects.constants';
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class ProjectsService extends BaseDbService<Project> {
|
export class ProjectsService extends BaseDbService<Project> {
|
||||||
@ -12,6 +17,7 @@ export class ProjectsService extends BaseDbService<Project> {
|
|||||||
constructor(
|
constructor(
|
||||||
@InjectRepository(Project)
|
@InjectRepository(Project)
|
||||||
readonly repository: Repository<Project>,
|
readonly repository: Repository<Project>,
|
||||||
|
private readonly amqpConnection: AmqpConnection,
|
||||||
) {
|
) {
|
||||||
super(repository);
|
super(repository);
|
||||||
}
|
}
|
||||||
@ -28,7 +34,12 @@ export class ProjectsService extends BaseDbService<Project> {
|
|||||||
async update(dto: UpdateProjectInput) {
|
async update(dto: UpdateProjectInput) {
|
||||||
await this.isDuplicateEntityForUpdate(dto.id, dto);
|
await this.isDuplicateEntityForUpdate(dto.id, dto);
|
||||||
const old = await this.findOne(dto.id);
|
const old = await this.findOne(dto.id);
|
||||||
return await this.repository.save(this.repository.merge(old, dto));
|
const project = await this.repository.save(this.repository.merge(old, dto));
|
||||||
|
this.amqpConnection.publish(EXCHANGE_PROJECT_FANOUT, ROUTE_PROJECT_CHANGE, [
|
||||||
|
project,
|
||||||
|
old,
|
||||||
|
]);
|
||||||
|
return project;
|
||||||
}
|
}
|
||||||
|
|
||||||
async remove(id: string) {
|
async remove(id: string) {
|
||||||
|
@ -3,3 +3,4 @@ export const ROUTE_FETCH = 'fetch';
|
|||||||
export const ROUTE_LIST_COMMITS = 'list-commits';
|
export const ROUTE_LIST_COMMITS = 'list-commits';
|
||||||
export const QUEUE_LIST_COMMITS = 'list-commits';
|
export const QUEUE_LIST_COMMITS = 'list-commits';
|
||||||
export const QUEUE_FETCH = 'repo-fetch';
|
export const QUEUE_FETCH = 'repo-fetch';
|
||||||
|
export const QUEUE_REFRESH_REPO = 'refresh-repo';
|
||||||
|
@ -7,12 +7,14 @@ import { ConfigModule, ConfigService } from '@nestjs/config';
|
|||||||
import { ProjectsModule } from '../projects/projects.module';
|
import { ProjectsModule } from '../projects/projects.module';
|
||||||
import { EXCHANGE_REPO } from './repos.constants';
|
import { EXCHANGE_REPO } from './repos.constants';
|
||||||
import { RabbitMQModule } from '@golevelup/nestjs-rabbitmq';
|
import { RabbitMQModule } from '@golevelup/nestjs-rabbitmq';
|
||||||
|
import { CommonsModule } from '../commons/commons.module';
|
||||||
|
|
||||||
@Module({
|
@Module({
|
||||||
imports: [
|
imports: [
|
||||||
TypeOrmModule.forFeature([Project]),
|
TypeOrmModule.forFeature([Project]),
|
||||||
ConfigModule,
|
ConfigModule,
|
||||||
ProjectsModule,
|
ProjectsModule,
|
||||||
|
CommonsModule,
|
||||||
RabbitMQModule.forRootAsync(RabbitMQModule, {
|
RabbitMQModule.forRootAsync(RabbitMQModule, {
|
||||||
imports: [ConfigModule],
|
imports: [ConfigModule],
|
||||||
useFactory: (configService: ConfigService) => ({
|
useFactory: (configService: ConfigService) => ({
|
||||||
|
@ -12,6 +12,7 @@ import { readFile } from 'fs/promises';
|
|||||||
import { getLoggerToken, PinoLogger } from 'nestjs-pino';
|
import { getLoggerToken, PinoLogger } from 'nestjs-pino';
|
||||||
import { Nack } from '@golevelup/nestjs-rabbitmq';
|
import { Nack } from '@golevelup/nestjs-rabbitmq';
|
||||||
import { getInstanceName } from '../commons/utils/rabbit-mq';
|
import { getInstanceName } from '../commons/utils/rabbit-mq';
|
||||||
|
import { RedisMutexService } from '../commons/redis-mutex/redis-mutex.service';
|
||||||
|
|
||||||
const getTest1Project = () =>
|
const getTest1Project = () =>
|
||||||
({
|
({
|
||||||
@ -52,6 +53,14 @@ describe('ReposService', () => {
|
|||||||
provide: getLoggerToken(ReposService.name),
|
provide: getLoggerToken(ReposService.name),
|
||||||
useValue: new PinoLogger({}),
|
useValue: new PinoLogger({}),
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
provide: RedisMutexService,
|
||||||
|
useValue: {
|
||||||
|
lock: jest.fn(() =>
|
||||||
|
Promise.resolve(() => Promise.resolve(undefined)),
|
||||||
|
),
|
||||||
|
},
|
||||||
|
},
|
||||||
],
|
],
|
||||||
}).compile();
|
}).compile();
|
||||||
|
|
||||||
|
@ -11,13 +11,14 @@ import { Project } from '../projects/project.entity';
|
|||||||
import { ListBranchesArgs } from './dtos/list-branches.args';
|
import { ListBranchesArgs } from './dtos/list-branches.args';
|
||||||
import { ConfigService } from '@nestjs/config';
|
import { ConfigService } from '@nestjs/config';
|
||||||
import { Commit } from './dtos/log-list.model';
|
import { Commit } from './dtos/log-list.model';
|
||||||
import { Nack, RabbitRPC } from '@golevelup/nestjs-rabbitmq';
|
import { Nack, RabbitRPC, RabbitSubscribe } from '@golevelup/nestjs-rabbitmq';
|
||||||
import { Pipeline } from '../pipelines/pipeline.entity';
|
import { Pipeline } from '../pipelines/pipeline.entity';
|
||||||
import { InjectPinoLogger, Logger } from 'nestjs-pino';
|
import { InjectPinoLogger, PinoLogger } from 'nestjs-pino';
|
||||||
import {
|
import {
|
||||||
EXCHANGE_REPO,
|
EXCHANGE_REPO,
|
||||||
QUEUE_FETCH,
|
QUEUE_FETCH,
|
||||||
QUEUE_LIST_COMMITS,
|
QUEUE_LIST_COMMITS,
|
||||||
|
QUEUE_REFRESH_REPO,
|
||||||
ROUTE_FETCH,
|
ROUTE_FETCH,
|
||||||
ROUTE_LIST_COMMITS,
|
ROUTE_LIST_COMMITS,
|
||||||
} from './repos.constants';
|
} from './repos.constants';
|
||||||
@ -27,6 +28,12 @@ import {
|
|||||||
getSelfInstanceRouteKey,
|
getSelfInstanceRouteKey,
|
||||||
} from '../commons/utils/rabbit-mq';
|
} from '../commons/utils/rabbit-mq';
|
||||||
import { ApplicationException } from '../commons/exceptions/application.exception';
|
import { ApplicationException } from '../commons/exceptions/application.exception';
|
||||||
|
import {
|
||||||
|
EXCHANGE_PROJECT_FANOUT,
|
||||||
|
ROUTE_PROJECT_CHANGE,
|
||||||
|
} from '../projects/projects.constants';
|
||||||
|
import { RedisMutexService } from '../commons/redis-mutex/redis-mutex.service';
|
||||||
|
import { rm } from 'fs/promises';
|
||||||
|
|
||||||
const DEFAULT_REMOTE_NAME = 'origin';
|
const DEFAULT_REMOTE_NAME = 'origin';
|
||||||
const INFO_PATH = '@info';
|
const INFO_PATH = '@info';
|
||||||
@ -37,7 +44,8 @@ export class ReposService {
|
|||||||
private readonly projectRepository: Repository<Project>,
|
private readonly projectRepository: Repository<Project>,
|
||||||
private readonly configService: ConfigService,
|
private readonly configService: ConfigService,
|
||||||
@InjectPinoLogger(ReposService.name)
|
@InjectPinoLogger(ReposService.name)
|
||||||
private readonly logger: Logger,
|
private readonly logger: PinoLogger,
|
||||||
|
private readonly redisMutexService: RedisMutexService,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
getWorkspaceRoot(project: Project): string {
|
getWorkspaceRoot(project: Project): string {
|
||||||
@ -170,6 +178,9 @@ export class ReposService {
|
|||||||
},
|
},
|
||||||
})
|
})
|
||||||
async fetch(pipeline: Pipeline): Promise<string | null | Nack> {
|
async fetch(pipeline: Pipeline): Promise<string | null | Nack> {
|
||||||
|
const unlock = await this.redisMutexService.lock(
|
||||||
|
`repo-project-${pipeline.projectId}`,
|
||||||
|
);
|
||||||
try {
|
try {
|
||||||
const git = await this.getGit(pipeline.project, undefined, {
|
const git = await this.getGit(pipeline.project, undefined, {
|
||||||
fetch: false,
|
fetch: false,
|
||||||
@ -179,6 +190,43 @@ export class ReposService {
|
|||||||
} catch (error) {
|
} catch (error) {
|
||||||
this.logger.error({ error, pipeline }, '[fetch] %s', error?.message);
|
this.logger.error({ error, pipeline }, '[fetch] %s', error?.message);
|
||||||
return new Nack();
|
return new Nack();
|
||||||
|
} finally {
|
||||||
|
await unlock();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@RabbitSubscribe({
|
||||||
|
exchange: EXCHANGE_PROJECT_FANOUT,
|
||||||
|
routingKey: ROUTE_PROJECT_CHANGE,
|
||||||
|
queue: QUEUE_REFRESH_REPO,
|
||||||
|
queueOptions: {
|
||||||
|
autoDelete: true,
|
||||||
|
durable: true,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
async refreshRepo([project]: [Project]) {
|
||||||
|
this.logger.info({ project }, '[refreshRepo] start');
|
||||||
|
const unlock = await this.redisMutexService.lock(
|
||||||
|
`repo-project-${project.id}`,
|
||||||
|
{
|
||||||
|
timeout: null,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
const path = join(
|
||||||
|
this.configService.get<string>('workspaces.root'),
|
||||||
|
encodeURIComponent(project.name),
|
||||||
|
);
|
||||||
|
await rm(path, { recursive: true });
|
||||||
|
this.logger.info({ project }, '[refreshRepo] success');
|
||||||
|
} catch (error) {
|
||||||
|
this.logger.error(
|
||||||
|
{ project, error },
|
||||||
|
'[refreshRepo] failed. $s',
|
||||||
|
error.message,
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
await unlock();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -7,6 +7,7 @@
|
|||||||
"experimentalDecorators": true,
|
"experimentalDecorators": true,
|
||||||
"allowSyntheticDefaultImports": true,
|
"allowSyntheticDefaultImports": true,
|
||||||
"target": "es2017",
|
"target": "es2017",
|
||||||
|
"lib": ["ES2021"],
|
||||||
"sourceMap": true,
|
"sourceMap": true,
|
||||||
"outDir": "./dist",
|
"outDir": "./dist",
|
||||||
"baseUrl": "./",
|
"baseUrl": "./",
|
||||||
|
Reference in New Issue
Block a user