feat(repos): 检出相关逻辑更改为传入任务信息而不是项目信息。
This commit is contained in:
parent
33b09594f5
commit
f39c801fc2
6
src/pipeline-tasks/models/work-unit-metadata.model.ts
Normal file
6
src/pipeline-tasks/models/work-unit-metadata.model.ts
Normal file
@ -0,0 +1,6 @@
|
|||||||
|
import { WorkUnit } from './work-unit.model';
|
||||||
|
|
||||||
|
export class WorkUnitMetadata {
|
||||||
|
version = 1;
|
||||||
|
units: WorkUnit[];
|
||||||
|
}
|
6
src/pipeline-tasks/models/work-unit.model.ts
Normal file
6
src/pipeline-tasks/models/work-unit.model.ts
Normal file
@ -0,0 +1,6 @@
|
|||||||
|
import { PipelineUnits as PipelineUnitTypes } from '../enums/pipeline-units.enum';
|
||||||
|
|
||||||
|
export class WorkUnit {
|
||||||
|
type: PipelineUnitTypes;
|
||||||
|
scripts: string[];
|
||||||
|
}
|
@ -1,5 +1,6 @@
|
|||||||
import { Test, TestingModule } from '@nestjs/testing';
|
import { Test, TestingModule } from '@nestjs/testing';
|
||||||
import { Job } from 'bull';
|
import { Job } from 'bull';
|
||||||
|
import { ReposService } from '../repos/repos.service';
|
||||||
import { PipelineUnits } from './enums/pipeline-units.enum';
|
import { PipelineUnits } from './enums/pipeline-units.enum';
|
||||||
import { PipelineTaskConsumer } from './pipeline-task.consumer';
|
import { PipelineTaskConsumer } from './pipeline-task.consumer';
|
||||||
import { PipelineTask } from './pipeline-task.entity';
|
import { PipelineTask } from './pipeline-task.entity';
|
||||||
@ -25,6 +26,12 @@ describe('PipelineTaskConsumer', () => {
|
|||||||
doNextTask: () => undefined,
|
doNextTask: () => undefined,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
provide: ReposService,
|
||||||
|
useValue: {
|
||||||
|
getWorkspaceRootByTask: () => 'workspace-root',
|
||||||
|
},
|
||||||
|
},
|
||||||
PipelineTaskConsumer,
|
PipelineTaskConsumer,
|
||||||
],
|
],
|
||||||
}).compile();
|
}).compile();
|
||||||
@ -45,4 +52,8 @@ describe('PipelineTaskConsumer', () => {
|
|||||||
expect(doNextTask).toHaveBeenCalledTimes(1);
|
expect(doNextTask).toHaveBeenCalledTimes(1);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe('doTask', () => {
|
||||||
|
it('should do all task', () => {});
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
@ -1,13 +1,57 @@
|
|||||||
|
import { ReposService } from './../repos/repos.service';
|
||||||
import { OnQueueCompleted, Process, Processor } from '@nestjs/bull';
|
import { OnQueueCompleted, Process, Processor } from '@nestjs/bull';
|
||||||
import { Job } from 'bull';
|
import { Job } from 'bull';
|
||||||
|
import { spawn } from 'child_process';
|
||||||
import { PipelineTask } from './pipeline-task.entity';
|
import { PipelineTask } from './pipeline-task.entity';
|
||||||
import { PIPELINE_TASK_QUEUE } from './pipeline-tasks.constants';
|
import { PIPELINE_TASK_QUEUE } from './pipeline-tasks.constants';
|
||||||
import { PipelineTasksService } from './pipeline-tasks.service';
|
import { PipelineTasksService } from './pipeline-tasks.service';
|
||||||
|
import { ApplicationException } from '../commons/exceptions/application.exception';
|
||||||
|
import { PipelineUnits } from './enums/pipeline-units.enum';
|
||||||
@Processor(PIPELINE_TASK_QUEUE)
|
@Processor(PIPELINE_TASK_QUEUE)
|
||||||
export class PipelineTaskConsumer {
|
export class PipelineTaskConsumer {
|
||||||
constructor(private readonly service: PipelineTasksService) {}
|
constructor(
|
||||||
|
private readonly service: PipelineTasksService,
|
||||||
|
private readonly reposService: ReposService,
|
||||||
|
) {}
|
||||||
@Process()
|
@Process()
|
||||||
async doTask() {}
|
async doTask({ data: task }: Job<PipelineTask>) {
|
||||||
|
const workspaceRoot = this.reposService.getWorkspaceRootByTask(task);
|
||||||
|
|
||||||
|
const units = task.units.map(
|
||||||
|
(type) =>
|
||||||
|
task.pipeline.workUnitMetadata.units.find(
|
||||||
|
(unit) => unit.type === type,
|
||||||
|
) ?? { type: type, scripts: [] },
|
||||||
|
);
|
||||||
|
|
||||||
|
for (const unit of units) {
|
||||||
|
// 检出代码时,不执行其他脚本。
|
||||||
|
if (unit.type === PipelineUnits.checkout) {
|
||||||
|
await this.reposService.checkout(task, workspaceRoot);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
for (const script of unit.scripts) {
|
||||||
|
await this.runScript(task, script, workspaceRoot);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async runScript(task: PipelineTask, script: string, workspaceRoot: string) {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
const errorMessages: string[] = [];
|
||||||
|
const sub = spawn(script, {
|
||||||
|
shell: true,
|
||||||
|
cwd: workspaceRoot,
|
||||||
|
});
|
||||||
|
sub.stderr.on('data', (data) => errorMessages.push(data));
|
||||||
|
sub.addListener('close', (code) => {
|
||||||
|
if (code === 0) {
|
||||||
|
return resolve(code);
|
||||||
|
}
|
||||||
|
return reject(new ApplicationException(errorMessages.join('\n')));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
@OnQueueCompleted()
|
@OnQueueCompleted()
|
||||||
onCompleted(job: Job<PipelineTask>) {
|
onCompleted(job: Job<PipelineTask>) {
|
||||||
|
@ -1,3 +1,4 @@
|
|||||||
|
import { AppBaseEntity } from './../commons/entities/app-base-entity';
|
||||||
import { ObjectType } from '@nestjs/graphql';
|
import { ObjectType } from '@nestjs/graphql';
|
||||||
import { Column, Entity, ManyToOne } from 'typeorm';
|
import { Column, Entity, ManyToOne } from 'typeorm';
|
||||||
import { Pipeline } from '../pipelines/pipeline.entity';
|
import { Pipeline } from '../pipelines/pipeline.entity';
|
||||||
@ -7,7 +8,7 @@ import { PipelineUnits } from './enums/pipeline-units.enum';
|
|||||||
|
|
||||||
@ObjectType()
|
@ObjectType()
|
||||||
@Entity()
|
@Entity()
|
||||||
export class PipelineTask {
|
export class PipelineTask extends AppBaseEntity {
|
||||||
@ManyToOne(() => Pipeline)
|
@ManyToOne(() => Pipeline)
|
||||||
pipeline: Pipeline;
|
pipeline: Pipeline;
|
||||||
@Column()
|
@Column()
|
||||||
|
@ -21,7 +21,10 @@ describe('PipelineTasksService', () => {
|
|||||||
id: 'test',
|
id: 'test',
|
||||||
name: '测试流水线',
|
name: '测试流水线',
|
||||||
branch: 'master',
|
branch: 'master',
|
||||||
workUnitMetadata: [],
|
workUnitMetadata: {},
|
||||||
|
project: {
|
||||||
|
id: 'test-project',
|
||||||
|
},
|
||||||
} as Pipeline);
|
} as Pipeline);
|
||||||
let redisClient;
|
let redisClient;
|
||||||
let taskQueue: Queue;
|
let taskQueue: Queue;
|
||||||
|
@ -2,6 +2,7 @@ import { Column, Entity, ManyToOne } from 'typeorm';
|
|||||||
import { AppBaseEntity } from '../commons/entities/app-base-entity';
|
import { AppBaseEntity } from '../commons/entities/app-base-entity';
|
||||||
import { Project } from '../projects/project.entity';
|
import { Project } from '../projects/project.entity';
|
||||||
import { ObjectType } from '@nestjs/graphql';
|
import { ObjectType } from '@nestjs/graphql';
|
||||||
|
import { WorkUnitMetadata } from '../pipeline-tasks/models/work-unit-metadata.model';
|
||||||
|
|
||||||
@ObjectType()
|
@ObjectType()
|
||||||
@Entity()
|
@Entity()
|
||||||
@ -18,5 +19,5 @@ export class Pipeline extends AppBaseEntity {
|
|||||||
name: string;
|
name: string;
|
||||||
|
|
||||||
@Column({ type: 'jsonb' })
|
@Column({ type: 'jsonb' })
|
||||||
workUnitMetadata: any;
|
workUnitMetadata: WorkUnitMetadata;
|
||||||
}
|
}
|
||||||
|
@ -1,12 +1,14 @@
|
|||||||
|
import { Pipeline } from './../pipelines/pipeline.entity';
|
||||||
import { Test, TestingModule } from '@nestjs/testing';
|
import { Test, TestingModule } from '@nestjs/testing';
|
||||||
import { getRepositoryToken } from '@nestjs/typeorm';
|
import { getRepositoryToken } from '@nestjs/typeorm';
|
||||||
import { Project } from '../projects/project.entity';
|
import { Project } from '../projects/project.entity';
|
||||||
import { ReposService } from './repos.service';
|
import { ReposService } from './repos.service';
|
||||||
import { ConfigModule } from '@nestjs/config';
|
import { ConfigModule } from '@nestjs/config';
|
||||||
import { readFile, rm } from 'fs/promises';
|
import { rm } from 'fs/promises';
|
||||||
import { join } from 'path';
|
|
||||||
import configuration from '../commons/config/configuration';
|
import configuration from '../commons/config/configuration';
|
||||||
import { NotFoundException } from '@nestjs/common';
|
import { PipelineTask } from '../pipeline-tasks/pipeline-task.entity';
|
||||||
|
import { join } from 'path';
|
||||||
|
import { readFile } from 'fs/promises';
|
||||||
|
|
||||||
const getTest1Project = () =>
|
const getTest1Project = () =>
|
||||||
({
|
({
|
||||||
@ -59,73 +61,79 @@ describe('ReposService', () => {
|
|||||||
it('getWorkspaceRoot', () => {
|
it('getWorkspaceRoot', () => {
|
||||||
expect(service.getWorkspaceRoot(getTest1Project())).toBeDefined();
|
expect(service.getWorkspaceRoot(getTest1Project())).toBeDefined();
|
||||||
});
|
});
|
||||||
describe.skip('listLogs', () => {
|
describe('listLogs', () => {
|
||||||
it('should be return logs', async () => {
|
it('should be return logs', async () => {
|
||||||
const result = await service.listLogs({ projectId: '1' });
|
const result = await service.listLogs({ projectId: '1' });
|
||||||
expect(result).toBeDefined();
|
expect(result).toBeDefined();
|
||||||
}, 20_000);
|
}, 20_000);
|
||||||
});
|
});
|
||||||
describe.skip('listBranch', () => {
|
describe('listBranch', () => {
|
||||||
it('should be return branches', async () => {
|
it('should be return branches', async () => {
|
||||||
const result = await service.listBranches({ projectId: '1' });
|
const result = await service.listBranches({ projectId: '1' });
|
||||||
expect(result).toBeDefined();
|
expect(result).toBeDefined();
|
||||||
}, 10_000);
|
}, 10_000);
|
||||||
});
|
});
|
||||||
describe.skip('checkoutBranch', () => {
|
|
||||||
it('should be checkout', async () => {
|
|
||||||
await service.checkoutBranch(getTest1Project(), 'master');
|
|
||||||
const filePath = join(
|
|
||||||
service.getWorkspaceRoot(getTest1Project(), 'master'),
|
|
||||||
'README.md',
|
|
||||||
);
|
|
||||||
const text = await readFile(filePath, { encoding: 'utf-8' });
|
|
||||||
expect(text).toMatch(/Commit 1/gi);
|
|
||||||
}, 30_000);
|
|
||||||
it('multiplexing workspace', async () => {
|
|
||||||
await service.checkoutBranch(getTest1Project(), 'master');
|
|
||||||
await service.checkoutBranch(getTest1Project(), 'branch-a');
|
|
||||||
await service.checkoutBranch(getTest1Project(), 'branch-b');
|
|
||||||
const filePath = join(
|
|
||||||
service.getWorkspaceRoot(getTest1Project(), 'branch-b'),
|
|
||||||
'branch-b.md',
|
|
||||||
);
|
|
||||||
const text = await readFile(filePath, { encoding: 'utf-8' });
|
|
||||||
expect(text).toMatch(/Commit branch b/gi);
|
|
||||||
}, 30_000);
|
|
||||||
it('nonexistent branch', async () => {
|
|
||||||
return expect(
|
|
||||||
service.checkoutBranch(getTest1Project(), 'nonexistent'),
|
|
||||||
).rejects.toBeInstanceOf(NotFoundException);
|
|
||||||
}, 30_000);
|
|
||||||
it('checkout the specified version', async () => {
|
|
||||||
await service.checkoutBranch(getTest1Project(), 'master');
|
|
||||||
const filePath = join(
|
|
||||||
service.getWorkspaceRoot(getTest1Project(), 'master'),
|
|
||||||
'README.md',
|
|
||||||
);
|
|
||||||
const text = await readFile(filePath, { encoding: 'utf-8' });
|
|
||||||
expect(text).toMatch(/Commit 1/gi);
|
|
||||||
}, 30_000);
|
|
||||||
});
|
|
||||||
|
|
||||||
describe.skip('checkoutCommit', () => {
|
describe.skip('checkout', () => {
|
||||||
|
let task: PipelineTask;
|
||||||
|
let workspaceRoot: string;
|
||||||
|
beforeEach(() => {
|
||||||
|
const project = new Project();
|
||||||
|
const pipeline = new Pipeline();
|
||||||
|
task = new PipelineTask();
|
||||||
|
pipeline.project = project;
|
||||||
|
task.pipeline = pipeline;
|
||||||
|
project.id = 'pid';
|
||||||
|
project.name = 'pname';
|
||||||
|
pipeline.id = 'lid';
|
||||||
|
pipeline.name = 'pipeline';
|
||||||
|
task.id = 'tid';
|
||||||
|
task.commit = '123123hash';
|
||||||
|
workspaceRoot = service.getWorkspaceRootByTask(task);
|
||||||
|
});
|
||||||
|
|
||||||
it('should be checkout', async () => {
|
it('should be checkout', async () => {
|
||||||
await service.checkoutCommit(getTest1Project(), '498c782685');
|
task.commit = '498c782685';
|
||||||
const filePath = join(
|
await service.checkout(task, workspaceRoot);
|
||||||
service.getWorkspaceRoot(getTest1Project(), '498c782685'),
|
const filePath = join(workspaceRoot, 'README.md');
|
||||||
'README.md',
|
|
||||||
);
|
|
||||||
const text = await readFile(filePath, { encoding: 'utf-8' });
|
const text = await readFile(filePath, { encoding: 'utf-8' });
|
||||||
expect(text).toMatch(/Commit 1/gi);
|
expect(text).toMatch(/Commit 1/gi);
|
||||||
}, 20_000);
|
}, 20_000);
|
||||||
it('should be checkout right commit', async () => {
|
it('should be checkout right commit', async () => {
|
||||||
await service.checkoutCommit(getTest1Project(), '7f7123fe5b');
|
task.commit = '7f7123fe5b';
|
||||||
const filePath = join(
|
await service.checkout(task, workspaceRoot);
|
||||||
service.getWorkspaceRoot(getTest1Project(), '7f7123fe5b'),
|
const filePath = join(workspaceRoot, 'README.md');
|
||||||
'README.md',
|
|
||||||
);
|
|
||||||
const text = await readFile(filePath, { encoding: 'utf-8' });
|
const text = await readFile(filePath, { encoding: 'utf-8' });
|
||||||
expect(text).toMatch(/(?!Commit 1)/gi);
|
expect(text).toMatch(/(?!Commit 1)/gi);
|
||||||
}, 20_000);
|
}, 20_000);
|
||||||
|
it('should be checkout right commit (复用)', async () => {
|
||||||
|
task.commit = '498c782685';
|
||||||
|
await service.checkout(task, workspaceRoot);
|
||||||
|
task.commit = '7f7123fe5b';
|
||||||
|
await service.checkout(task, workspaceRoot);
|
||||||
|
const filePath = join(workspaceRoot, 'README.md');
|
||||||
|
const text = await readFile(filePath, { encoding: 'utf-8' });
|
||||||
|
expect(text).toMatch(/(?!Commit 1)/gi);
|
||||||
|
}, 30_000);
|
||||||
|
});
|
||||||
|
|
||||||
|
describe('getWorkspaceRootByTask', () => {
|
||||||
|
it('should be return right path', () => {
|
||||||
|
const project = new Project();
|
||||||
|
const pipeline = new Pipeline();
|
||||||
|
const task = new PipelineTask();
|
||||||
|
pipeline.project = project;
|
||||||
|
task.pipeline = pipeline;
|
||||||
|
project.id = 'pid';
|
||||||
|
project.name = 'pname';
|
||||||
|
pipeline.id = 'lid';
|
||||||
|
pipeline.name = 'pipeline/\\-名称';
|
||||||
|
task.id = 'tid';
|
||||||
|
task.commit = '123123hash';
|
||||||
|
|
||||||
|
expect(service.getWorkspaceRootByTask(task)).toMatch(
|
||||||
|
/\/pname\/pipeline%2F%5C-%E5%90%8D%E7%A7%B0-123123hash$/,
|
||||||
|
);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
@ -1,3 +1,5 @@
|
|||||||
|
import { Pipeline } from './../pipelines/pipeline.entity';
|
||||||
|
import { PipelineTask } from './../pipeline-tasks/pipeline-task.entity';
|
||||||
import { Injectable, NotFoundException } from '@nestjs/common';
|
import { Injectable, NotFoundException } from '@nestjs/common';
|
||||||
import { InjectRepository } from '@nestjs/typeorm';
|
import { InjectRepository } from '@nestjs/typeorm';
|
||||||
import { F_OK } from 'constants';
|
import { F_OK } from 'constants';
|
||||||
@ -11,6 +13,7 @@ import { ListLogsArgs } from './dtos/list-logs.args';
|
|||||||
import { ConfigService } from '@nestjs/config';
|
import { ConfigService } from '@nestjs/config';
|
||||||
|
|
||||||
const DEFAULT_REMOTE_NAME = 'origin';
|
const DEFAULT_REMOTE_NAME = 'origin';
|
||||||
|
const INFO_PATH = '@info';
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class ReposService {
|
export class ReposService {
|
||||||
constructor(
|
constructor(
|
||||||
@ -19,33 +22,28 @@ export class ReposService {
|
|||||||
private readonly configService: ConfigService,
|
private readonly configService: ConfigService,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
getWorkspaceRoot(project: Project, subDir = ''): string {
|
getWorkspaceRoot(project: Project): string {
|
||||||
return join(
|
return join(
|
||||||
this.configService.get<string>('workspaces.root'),
|
this.configService.get<string>('workspaces.root'),
|
||||||
project.name,
|
encodeURIComponent(project.name),
|
||||||
encodeURIComponent(subDir),
|
INFO_PATH,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
async lockWorkspace(workspaceRoot: string) {
|
async getGit(project: Project, workspaceRoot?: string) {
|
||||||
// TODO: 获取锁,失败抛错。
|
if (!workspaceRoot) {
|
||||||
}
|
workspaceRoot = this.getWorkspaceRoot(project);
|
||||||
|
}
|
||||||
|
|
||||||
async getGit(project: Project, subDir = 'default') {
|
await access(workspaceRoot, F_OK).catch(async () => {
|
||||||
const workspaceRoot = this.getWorkspaceRoot(project, subDir);
|
await mkdir(workspaceRoot, { recursive: true });
|
||||||
await this.lockWorkspace(workspaceRoot);
|
});
|
||||||
|
|
||||||
const firstInit = await access(workspaceRoot, F_OK)
|
|
||||||
.then(() => false)
|
|
||||||
.catch(async () => {
|
|
||||||
await mkdir(workspaceRoot, { recursive: true });
|
|
||||||
return true;
|
|
||||||
});
|
|
||||||
const git = gitP(workspaceRoot);
|
const git = gitP(workspaceRoot);
|
||||||
if (firstInit) {
|
if (!(await git.checkIsRepo())) {
|
||||||
await git.init();
|
await git.init();
|
||||||
await git.addRemote(DEFAULT_REMOTE_NAME, project.sshUrl);
|
await git.addRemote(DEFAULT_REMOTE_NAME, project.sshUrl);
|
||||||
}
|
}
|
||||||
|
await git.fetch();
|
||||||
return git;
|
return git;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -54,7 +52,6 @@ export class ReposService {
|
|||||||
id: dto.projectId,
|
id: dto.projectId,
|
||||||
});
|
});
|
||||||
const git = await this.getGit(project);
|
const git = await this.getGit(project);
|
||||||
await git.fetch();
|
|
||||||
return await git.log(
|
return await git.log(
|
||||||
dto.branch ? ['--branches', dto.branch, '--'] : ['--all'],
|
dto.branch ? ['--branches', dto.branch, '--'] : ['--all'],
|
||||||
);
|
);
|
||||||
@ -68,26 +65,8 @@ export class ReposService {
|
|||||||
return git.branch();
|
return git.branch();
|
||||||
}
|
}
|
||||||
|
|
||||||
async checkoutBranch(project: Project, branch: string) {
|
async checkout(task: PipelineTask, workspaceRoot: string) {
|
||||||
const git = await this.getGit(project, branch);
|
const git = await this.getGit(task.pipeline.project, workspaceRoot);
|
||||||
try {
|
|
||||||
await git.fetch(DEFAULT_REMOTE_NAME, branch);
|
|
||||||
} catch (err) {
|
|
||||||
if (err.message.includes("couldn't find remote ref nonexistent")) {
|
|
||||||
throw new NotFoundException(err.message);
|
|
||||||
}
|
|
||||||
throw err;
|
|
||||||
}
|
|
||||||
await git.checkout([
|
|
||||||
'-B',
|
|
||||||
branch,
|
|
||||||
'--track',
|
|
||||||
`${DEFAULT_REMOTE_NAME}/${branch}`,
|
|
||||||
]);
|
|
||||||
}
|
|
||||||
|
|
||||||
async checkoutCommit(project: Project, commitNumber: string) {
|
|
||||||
const git = await this.getGit(project, commitNumber);
|
|
||||||
try {
|
try {
|
||||||
await git.fetch(DEFAULT_REMOTE_NAME);
|
await git.fetch(DEFAULT_REMOTE_NAME);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
@ -96,6 +75,20 @@ export class ReposService {
|
|||||||
}
|
}
|
||||||
throw err;
|
throw err;
|
||||||
}
|
}
|
||||||
await git.checkout([commitNumber]);
|
await git.checkout([task.commit]);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* get workspace root absolute path
|
||||||
|
*
|
||||||
|
* ! example: `/var/tmp/fennec-workspaces-root/project/pipeline_name-commit_hash`
|
||||||
|
* @param task {PipelineTask} task (with pipeline and project info)
|
||||||
|
*/
|
||||||
|
getWorkspaceRootByTask(task: PipelineTask) {
|
||||||
|
return join(
|
||||||
|
this.configService.get<string>('workspaces.root'),
|
||||||
|
encodeURIComponent(task.pipeline.project.name),
|
||||||
|
encodeURIComponent(`${task.pipeline.name}-${task.commit}`),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user