This is page 3 of 3. Use http://codebase.md/circleci-public/mcp-server-circleci?page={x} to view the full context.
# Directory Structure
```
├── .circleci
│ └── config.yml
├── .dockerignore
├── .github
│ ├── CODEOWNERS
│ ├── ISSUE_TEMPLATE
│ │ ├── BUG.yml
│ │ └── FEATURE_REQUEST.yml
│ └── PULL_REQUEST_TEMPLATE
│ └── PULL_REQUEST.md
├── .gitignore
├── .npmrc
├── .nvmrc
├── .prettierignore
├── .prettierrc
├── CHANGELOG.md
├── CODE_OF_CONDUCT.md
├── CONTRIBUTING.md
├── Dockerfile
├── eslint.config.js
├── LICENSE
├── package.json
├── pnpm-lock.yaml
├── README.md
├── renovate.json
├── scripts
│ └── create-tool.js
├── smithery.yaml
├── src
│ ├── circleci-tools.ts
│ ├── clients
│ │ ├── circleci
│ │ │ ├── configValidate.ts
│ │ │ ├── deploys.ts
│ │ │ ├── httpClient.test.ts
│ │ │ ├── httpClient.ts
│ │ │ ├── index.ts
│ │ │ ├── insights.ts
│ │ │ ├── jobs.ts
│ │ │ ├── jobsV1.ts
│ │ │ ├── pipelines.ts
│ │ │ ├── projects.ts
│ │ │ ├── tests.ts
│ │ │ ├── usage.ts
│ │ │ └── workflows.ts
│ │ ├── circleci-private
│ │ │ ├── index.ts
│ │ │ ├── jobsPrivate.ts
│ │ │ └── me.ts
│ │ ├── circlet
│ │ │ ├── circlet.ts
│ │ │ └── index.ts
│ │ ├── client.ts
│ │ └── schemas.ts
│ ├── index.ts
│ ├── lib
│ │ ├── flaky-tests
│ │ │ └── getFlakyTests.ts
│ │ ├── getWorkflowIdFromURL.test.ts
│ │ ├── getWorkflowIdFromURL.ts
│ │ ├── latest-pipeline
│ │ │ ├── formatLatestPipelineStatus.ts
│ │ │ └── getLatestPipelineWorkflows.ts
│ │ ├── mcpErrorOutput.test.ts
│ │ ├── mcpErrorOutput.ts
│ │ ├── mcpResponse.test.ts
│ │ ├── mcpResponse.ts
│ │ ├── outputTextTruncated.test.ts
│ │ ├── outputTextTruncated.ts
│ │ ├── pipeline-job-logs
│ │ │ ├── getJobLogs.ts
│ │ │ └── getPipelineJobLogs.ts
│ │ ├── pipeline-job-tests
│ │ │ ├── formatJobTests.ts
│ │ │ └── getJobTests.ts
│ │ ├── project-detection
│ │ │ ├── index.test.ts
│ │ │ ├── index.ts
│ │ │ └── vcsTool.ts
│ │ ├── rateLimitedRequests
│ │ │ ├── index.test.ts
│ │ │ └── index.ts
│ │ └── usage-api
│ │ ├── findUnderusedResourceClasses.test.ts
│ │ ├── findUnderusedResourceClasses.ts
│ │ ├── getUsageApiData.test.ts
│ │ ├── getUsageApiData.ts
│ │ └── parseDateTimeString.ts
│ ├── tools
│ │ ├── analyzeDiff
│ │ │ ├── handler.test.ts
│ │ │ ├── handler.ts
│ │ │ ├── inputSchema.ts
│ │ │ └── tool.ts
│ │ ├── configHelper
│ │ │ ├── handler.test.ts
│ │ │ ├── handler.ts
│ │ │ ├── inputSchema.ts
│ │ │ └── tool.ts
│ │ ├── createPromptTemplate
│ │ │ ├── handler.test.ts
│ │ │ ├── handler.ts
│ │ │ ├── inputSchema.ts
│ │ │ └── tool.ts
│ │ ├── downloadUsageApiData
│ │ │ ├── handler.test.ts
│ │ │ ├── handler.ts
│ │ │ ├── inputSchema.ts
│ │ │ └── tool.ts
│ │ ├── findUnderusedResourceClasses
│ │ │ ├── handler.test.ts
│ │ │ ├── handler.ts
│ │ │ ├── inputSchema.ts
│ │ │ └── tool.ts
│ │ ├── getBuildFailureLogs
│ │ │ ├── handler.test.ts
│ │ │ ├── handler.ts
│ │ │ ├── inputSchema.ts
│ │ │ └── tool.ts
│ │ ├── getFlakyTests
│ │ │ ├── handler.test.ts
│ │ │ ├── handler.ts
│ │ │ ├── inputSchema.ts
│ │ │ └── tool.ts
│ │ ├── getJobTestResults
│ │ │ ├── handler.test.ts
│ │ │ ├── handler.ts
│ │ │ ├── inputSchema.ts
│ │ │ └── tool.ts
│ │ ├── getLatestPipelineStatus
│ │ │ ├── handler.test.ts
│ │ │ ├── handler.ts
│ │ │ ├── inputSchema.ts
│ │ │ └── tool.ts
│ │ ├── listComponentVersions
│ │ │ ├── handler.test.ts
│ │ │ ├── handler.ts
│ │ │ ├── inputSchema.ts
│ │ │ └── tool.ts
│ │ ├── listFollowedProjects
│ │ │ ├── handler.test.ts
│ │ │ ├── handler.ts
│ │ │ ├── inputSchema.ts
│ │ │ └── tool.ts
│ │ ├── recommendPromptTemplateTests
│ │ │ ├── handler.test.ts
│ │ │ ├── handler.ts
│ │ │ ├── inputSchema.ts
│ │ │ └── tool.ts
│ │ ├── rerunWorkflow
│ │ │ ├── handler.test.ts
│ │ │ ├── handler.ts
│ │ │ ├── inputSchema.ts
│ │ │ └── tool.ts
│ │ ├── runEvaluationTests
│ │ │ ├── handler.test.ts
│ │ │ ├── handler.ts
│ │ │ ├── inputSchema.ts
│ │ │ └── tool.ts
│ │ ├── runPipeline
│ │ │ ├── handler.test.ts
│ │ │ ├── handler.ts
│ │ │ ├── inputSchema.ts
│ │ │ └── tool.ts
│ │ ├── runRollbackPipeline
│ │ │ ├── handler.test.ts
│ │ │ ├── handler.ts
│ │ │ ├── inputSchema.ts
│ │ │ └── tool.ts
│ │ └── shared
│ │ └── constants.ts
│ └── transports
│ ├── stdio.ts
│ └── unified.ts
├── tsconfig.json
├── tsconfig.test.json
└── vitest.config.js
```
# Files
--------------------------------------------------------------------------------
/src/tools/analyzeDiff/handler.test.ts:
--------------------------------------------------------------------------------
```typescript
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { FilterBy } from '../shared/constants.js';
import { analyzeDiff } from './handler.js';
import { analyzeDiffInputSchema } from './inputSchema.js';
import { CircletClient } from '../../clients/circlet/index.js';
import { RuleReview } from '../../clients/schemas.js';
// Mock the CircletClient
vi.mock('../../clients/circlet/index.js');
describe('analyzeDiff', () => {
beforeEach(() => {
vi.clearAllMocks();
});
it('should return no rules message when rules is an empty string', async () => {
const mockCircletInstance = {
circlet: {
ruleReview: vi.fn(),
},
};
vi.mocked(CircletClient).mockImplementation(
() => mockCircletInstance as any,
);
const mockArgs = {
params: {
speedMode: false,
filterBy: FilterBy.none,
diff: 'diff --git a/test.ts b/test.ts\n+console.log("test");',
rules: '',
},
};
const controller = new AbortController();
const result = await analyzeDiff(mockArgs, { signal: controller.signal });
expect(result).toEqual({
content: [
{
type: 'text',
text: 'No rules found. Please add rules to your repository.',
},
],
});
});
it('should return no diff message when diff is an empty string', async () => {
const mockCircletInstance = {
circlet: {
ruleReview: vi.fn(),
},
};
vi.mocked(CircletClient).mockImplementation(
() => mockCircletInstance as any,
);
const mockArgs = {
params: {
speedMode: false,
filterBy: FilterBy.none,
diff: '',
rules: '',
},
};
const controller = new AbortController();
const result = await analyzeDiff(mockArgs, { signal: controller.signal });
expect(result).toEqual({
content: [
{
type: 'text',
text: 'No diff found. Please provide a diff to analyze.',
},
],
});
});
it('should handle complex diff content with multiple rules', async () => {
const mockRuleReview: RuleReview = {
isRuleCompliant: true,
relatedRules: {
compliant: [
{
rule: 'Rule 1: No console.log statements',
reason: 'No console.log statements found',
confidenceScore: 0.95,
},
],
violations: [],
requiresHumanReview: [],
},
unrelatedRules: [],
};
const mockCircletInstance = {
circlet: {
ruleReview: vi.fn().mockResolvedValue(mockRuleReview),
},
};
vi.mocked(CircletClient).mockImplementation(
() => mockCircletInstance as any,
);
const mockArgs = {
params: {
speedMode: false,
filterBy: FilterBy.none,
diff: `diff --git a/src/component.ts b/src/component.ts
index 1234567..abcdefg 100644
--- a/src/component.ts
+++ b/src/component.ts
@@ -1,5 +1,8 @@
export class Component {
+ private data: any = {};
+
constructor() {
+ console.log("Component created");
}
}`,
rules: `Rule 1: No console.log statements
Rule 2: Avoid using 'any' type
Rule 3: Use proper TypeScript types
---
Rule 4: All functions must have JSDoc comments`,
},
};
const controller = new AbortController();
const result = await analyzeDiff(mockArgs, { signal: controller.signal });
expect(mockCircletInstance.circlet.ruleReview).toHaveBeenCalledWith({
speedMode: false,
filterBy: FilterBy.none,
diff: mockArgs.params.diff,
rules: mockArgs.params.rules,
});
expect(result).toEqual({
content: [
{
type: 'text',
text: 'All rules are compliant.',
},
],
});
});
it('should handle multiline rules and preserve formatting', async () => {
const mockRuleReview: RuleReview = {
isRuleCompliant: false,
relatedRules: {
compliant: [],
violations: [
{
rule: 'No Console Logs',
reason: 'Console.log statements found in code',
confidenceScore: 0.98,
violationInstances: [
{
file: 'src/component.ts',
lineNumbersInDiff: ['2'],
violatingCodeSnippet: 'console.log(x);',
explanationOfViolation: 'Direct console.log usage',
},
],
},
],
requiresHumanReview: [],
},
unrelatedRules: [],
};
const mockCircletInstance = {
circlet: {
ruleReview: vi.fn().mockResolvedValue(mockRuleReview),
},
};
vi.mocked(CircletClient).mockImplementation(
() => mockCircletInstance as any,
);
const mockArgs = {
params: {
speedMode: false,
filterBy: FilterBy.none,
diff: '+const x = 5;\n+console.log(x);',
rules: `# IDE Rules Example
## Rule: No Console Logs
Description: Remove all console.log statements before committing code.
## Rule: TypeScript Safety
Description: Avoid using 'any' type.`,
},
};
const controller = new AbortController();
const result = await analyzeDiff(mockArgs, { signal: controller.signal });
expect(mockCircletInstance.circlet.ruleReview).toHaveBeenCalledWith({
speedMode: false,
filterBy: FilterBy.none,
diff: mockArgs.params.diff,
rules: mockArgs.params.rules,
});
expect(result.content[0].type).toBe('text');
expect(result.content[0].text).toContain('Rule: No Console Logs');
expect(result.content[0].text).toContain(
'Reason: Console.log statements found in code',
);
expect(result.content[0].text).toContain('Confidence Score: 0.98');
});
it('should return compliant message when all rules are followed', async () => {
const mockRuleReview: RuleReview = {
isRuleCompliant: true,
relatedRules: {
compliant: [
{
rule: 'No console.log statements',
reason: 'Code follows proper logging practices',
confidenceScore: 0.95,
},
],
violations: [],
requiresHumanReview: [],
},
unrelatedRules: [],
};
const mockCircletInstance = {
circlet: {
ruleReview: vi.fn().mockResolvedValue(mockRuleReview),
},
};
vi.mocked(CircletClient).mockImplementation(
() => mockCircletInstance as any,
);
const mockArgs = {
params: {
speedMode: false,
filterBy: FilterBy.none,
diff: 'diff --git a/test.ts b/test.ts\n+const logger = new Logger();',
rules: 'Rule 1: No console.log statements\nRule 2: Use proper logging',
},
};
const controller = new AbortController();
const result = await analyzeDiff(mockArgs, { signal: controller.signal });
expect(mockCircletInstance.circlet.ruleReview).toHaveBeenCalledWith({
speedMode: false,
filterBy: FilterBy.none,
diff: mockArgs.params.diff,
rules: mockArgs.params.rules,
});
expect(result).toEqual({
content: [
{
type: 'text',
text: 'All rules are compliant.',
},
],
});
});
it('should return formatted violations when rules are violated', async () => {
const mockRuleReview: RuleReview = {
isRuleCompliant: false,
relatedRules: {
compliant: [],
violations: [
{
rule: 'No console.log statements',
reason: 'Console.log statements found in the code',
confidenceScore: 0.98,
violationInstances: [
{
file: 'src/component.ts',
lineNumbersInDiff: ['5'],
violatingCodeSnippet: 'console.log("test");',
explanationOfViolation: 'Direct console.log usage',
},
],
},
{
rule: 'Avoid using any type',
reason: 'Any type usage reduces type safety',
confidenceScore: 0.92,
violationInstances: [
{
file: 'src/component.ts',
lineNumbersInDiff: ['3'],
violatingCodeSnippet: 'private data: any = {};',
explanationOfViolation: 'Variable declared with any type',
},
],
},
],
requiresHumanReview: [],
},
unrelatedRules: [],
};
const mockCircletInstance = {
circlet: {
ruleReview: vi.fn().mockResolvedValue(mockRuleReview),
},
};
vi.mocked(CircletClient).mockImplementation(
() => mockCircletInstance as any,
);
const mockArgs = {
params: {
speedMode: false,
filterBy: FilterBy.none,
diff: `diff --git a/src/component.ts b/src/component.ts
index 1234567..abcdefg 100644
--- a/src/component.ts
+++ b/src/component.ts
@@ -1,5 +1,8 @@
export class Component {
+ private data: any = {};
+
constructor() {
+ console.log("Component created");
}
}`,
rules: `Rule 1: No console.log statements
Rule 2: Avoid using 'any' type
Rule 3: Use proper TypeScript types`,
},
};
const controller = new AbortController();
const result = await analyzeDiff(mockArgs, { signal: controller.signal });
expect(mockCircletInstance.circlet.ruleReview).toHaveBeenCalledWith({
filterBy: FilterBy.none,
speedMode: false,
diff: mockArgs.params.diff,
rules: mockArgs.params.rules,
});
expect(result).toEqual({
content: [
{
type: 'text',
text: `Rule: No console.log statements
Reason: Console.log statements found in the code
Confidence Score: 0.98
Rule: Avoid using any type
Reason: Any type usage reduces type safety
Confidence Score: 0.92`,
},
],
});
});
it('should handle single violation correctly', async () => {
const mockRuleReview: RuleReview = {
isRuleCompliant: false,
relatedRules: {
compliant: [],
violations: [
{
rule: 'No magic numbers',
reason: 'Magic numbers make code less maintainable',
confidenceScore: 0.85,
violationInstances: [
{
file: 'src/component.ts',
lineNumbersInDiff: ['2'],
violatingCodeSnippet: 'const timeout = 5000;',
explanationOfViolation: 'Hardcoded timeout value',
},
],
},
],
requiresHumanReview: [],
},
unrelatedRules: [],
};
const mockCircletInstance = {
circlet: {
ruleReview: vi.fn().mockResolvedValue(mockRuleReview),
},
};
vi.mocked(CircletClient).mockImplementation(
() => mockCircletInstance as any,
);
const mockArgs = {
params: {
speedMode: false,
filterBy: FilterBy.none,
diff: '+const timeout = 5000;',
rules: 'Rule: No magic numbers',
},
};
const controller = new AbortController();
const result = await analyzeDiff(mockArgs, { signal: controller.signal });
expect(result).toEqual({
content: [
{
type: 'text',
text: `Rule: No magic numbers
Reason: Magic numbers make code less maintainable
Confidence Score: 0.85`,
},
],
});
});
it('should set default values for speedMode and filterBy when not provided', async () => {
const mockRuleReview: RuleReview = {
isRuleCompliant: true,
relatedRules: {
compliant: [],
violations: [],
requiresHumanReview: [],
},
unrelatedRules: [],
};
const mockCircletInstance = {
circlet: {
ruleReview: vi.fn().mockResolvedValue(mockRuleReview),
},
};
vi.mocked(CircletClient).mockImplementation(
() => mockCircletInstance as any,
);
const rawParams = {
diff: '+const timeout = 5000;',
rules: 'Rule: No magic numbers',
};
const parsedParams = analyzeDiffInputSchema.parse(rawParams);
const mockArgs = {
params: parsedParams,
};
const controller = new AbortController();
await analyzeDiff(mockArgs, { signal: controller.signal });
// Verify default values (filterBy: FilterBy.none & speedMode: false) are applied when not explictly stated
expect(mockCircletInstance.circlet.ruleReview).toHaveBeenCalledWith({
diff: rawParams.diff,
rules: rawParams.rules,
filterBy: FilterBy.none,
speedMode: false,
});
});
});
```
--------------------------------------------------------------------------------
/src/tools/runPipeline/handler.test.ts:
--------------------------------------------------------------------------------
```typescript
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { runPipeline } from './handler.js';
import * as projectDetection from '../../lib/project-detection/index.js';
import * as clientModule from '../../clients/client.js';
vi.mock('../../lib/project-detection/index.js');
vi.mock('../../clients/client.js');
describe('runPipeline handler', () => {
const mockCircleCIClient = {
projects: {
getProject: vi.fn(),
},
pipelines: {
getPipelineDefinitions: vi.fn(),
runPipeline: vi.fn(),
},
};
beforeEach(() => {
vi.resetAllMocks();
vi.spyOn(clientModule, 'getCircleCIClient').mockReturnValue(
mockCircleCIClient as any,
);
});
it('should return a valid MCP error response when no inputs are provided', async () => {
const args = {
params: {},
} as any;
const controller = new AbortController();
const response = await runPipeline(args, {
signal: controller.signal,
});
expect(response).toHaveProperty('content');
expect(response).toHaveProperty('isError', true);
expect(Array.isArray(response.content)).toBe(true);
expect(response.content[0]).toHaveProperty('type', 'text');
expect(typeof response.content[0].text).toBe('string');
});
it('should return a valid MCP error response when project is not found', async () => {
vi.spyOn(projectDetection, 'identifyProjectSlug').mockResolvedValue(
undefined,
);
const args = {
params: {
workspaceRoot: '/workspace',
gitRemoteURL: 'https://github.com/org/repo.git',
branch: 'main',
},
} as any;
const controller = new AbortController();
const response = await runPipeline(args, {
signal: controller.signal,
});
expect(response).toHaveProperty('content');
expect(response).toHaveProperty('isError', true);
expect(Array.isArray(response.content)).toBe(true);
expect(response.content[0]).toHaveProperty('type', 'text');
expect(typeof response.content[0].text).toBe('string');
});
it('should return a valid MCP error response when no branch is provided', async () => {
vi.spyOn(projectDetection, 'getProjectSlugFromURL').mockReturnValue(
'gh/org/repo',
);
vi.spyOn(projectDetection, 'getBranchFromURL').mockReturnValue(undefined);
const args = {
params: {
projectURL: 'https://app.circleci.com/pipelines/gh/org/repo',
},
} as any;
const controller = new AbortController();
const response = await runPipeline(args, {
signal: controller.signal,
});
expect(response).toHaveProperty('content');
expect(response).toHaveProperty('isError', true);
expect(Array.isArray(response.content)).toBe(true);
expect(response.content[0]).toHaveProperty('type', 'text');
expect(typeof response.content[0].text).toBe('string');
expect(response.content[0].text).toContain('No branch provided');
});
it('should return a valid MCP error response when projectSlug is provided without branch', async () => {
const args = {
params: {
projectSlug: 'gh/org/repo',
// No branch provided
},
} as any;
const controller = new AbortController();
const response = await runPipeline(args, {
signal: controller.signal,
});
expect(response).toHaveProperty('content');
expect(response).toHaveProperty('isError', true);
expect(Array.isArray(response.content)).toBe(true);
expect(response.content[0]).toHaveProperty('type', 'text');
expect(response.content[0].text).toContain('Branch not provided');
// Verify that CircleCI API was not called
expect(mockCircleCIClient.projects.getProject).not.toHaveBeenCalled();
expect(
mockCircleCIClient.pipelines.getPipelineDefinitions,
).not.toHaveBeenCalled();
expect(mockCircleCIClient.pipelines.runPipeline).not.toHaveBeenCalled();
});
it('should return a valid MCP error response when no pipeline definitions are found', async () => {
vi.spyOn(projectDetection, 'getProjectSlugFromURL').mockReturnValue(
'gh/org/repo',
);
vi.spyOn(projectDetection, 'getBranchFromURL').mockReturnValue('main');
mockCircleCIClient.projects.getProject.mockResolvedValue({
id: 'project-id',
});
mockCircleCIClient.pipelines.getPipelineDefinitions.mockResolvedValue([]);
const args = {
params: {
projectURL: 'https://app.circleci.com/pipelines/gh/org/repo',
},
} as any;
const controller = new AbortController();
const response = await runPipeline(args, {
signal: controller.signal,
});
expect(response).toHaveProperty('content');
expect(response).toHaveProperty('isError', true);
expect(Array.isArray(response.content)).toBe(true);
expect(response.content[0]).toHaveProperty('type', 'text');
expect(typeof response.content[0].text).toBe('string');
expect(response.content[0].text).toContain('No pipeline definitions found');
});
it('should return a list of pipeline choices when multiple pipeline definitions are found and no choice is provided', async () => {
vi.spyOn(projectDetection, 'getProjectSlugFromURL').mockReturnValue(
'gh/org/repo',
);
vi.spyOn(projectDetection, 'getBranchFromURL').mockReturnValue('main');
mockCircleCIClient.projects.getProject.mockResolvedValue({
id: 'project-id',
});
mockCircleCIClient.pipelines.getPipelineDefinitions.mockResolvedValue([
{ id: 'def1', name: 'Pipeline 1' },
{ id: 'def2', name: 'Pipeline 2' },
]);
const args = {
params: {
projectURL: 'https://app.circleci.com/pipelines/gh/org/repo',
},
} as any;
const controller = new AbortController();
const response = await runPipeline(args, {
signal: controller.signal,
});
expect(response).toHaveProperty('content');
expect(Array.isArray(response.content)).toBe(true);
expect(response.content[0]).toHaveProperty('type', 'text');
expect(typeof response.content[0].text).toBe('string');
expect(response.content[0].text).toContain(
'Multiple pipeline definitions found',
);
expect(response.content[0].text).toContain('Pipeline 1');
expect(response.content[0].text).toContain('Pipeline 2');
});
it('should return an error when an invalid pipeline choice is provided', async () => {
vi.spyOn(projectDetection, 'getProjectSlugFromURL').mockReturnValue(
'gh/org/repo',
);
vi.spyOn(projectDetection, 'getBranchFromURL').mockReturnValue('main');
mockCircleCIClient.projects.getProject.mockResolvedValue({
id: 'project-id',
});
mockCircleCIClient.pipelines.getPipelineDefinitions.mockResolvedValue([
{ id: 'def1', name: 'Pipeline 1' },
{ id: 'def2', name: 'Pipeline 2' },
]);
const args = {
params: {
projectURL: 'https://app.circleci.com/pipelines/gh/org/repo',
pipelineChoiceName: 'Non-existent Pipeline',
},
} as any;
const controller = new AbortController();
const response = await runPipeline(args, {
signal: controller.signal,
});
expect(response).toHaveProperty('content');
expect(response).toHaveProperty('isError', true);
expect(Array.isArray(response.content)).toBe(true);
expect(response.content[0]).toHaveProperty('type', 'text');
expect(typeof response.content[0].text).toBe('string');
expect(response.content[0].text).toContain(
'Pipeline definition with name Non-existent Pipeline not found',
);
});
it('should run a pipeline with a specific choice when valid pipeline choice is provided', async () => {
vi.spyOn(projectDetection, 'getProjectSlugFromURL').mockReturnValue(
'gh/org/repo',
);
vi.spyOn(projectDetection, 'getBranchFromURL').mockReturnValue('main');
mockCircleCIClient.projects.getProject.mockResolvedValue({
id: 'project-id',
});
mockCircleCIClient.pipelines.getPipelineDefinitions.mockResolvedValue([
{ id: 'def1', name: 'Pipeline 1' },
{ id: 'def2', name: 'Pipeline 2' },
]);
mockCircleCIClient.pipelines.runPipeline.mockResolvedValue({
number: 123,
state: 'pending',
id: 'pipeline-id',
});
const args = {
params: {
projectURL: 'https://app.circleci.com/pipelines/gh/org/repo',
pipelineChoiceName: 'Pipeline 2',
},
} as any;
const controller = new AbortController();
const response = await runPipeline(args, {
signal: controller.signal,
});
expect(response).toHaveProperty('content');
expect(Array.isArray(response.content)).toBe(true);
expect(response.content[0]).toHaveProperty('type', 'text');
expect(typeof response.content[0].text).toBe('string');
expect(response.content[0].text).toContain('Pipeline run successfully');
expect(mockCircleCIClient.pipelines.runPipeline).toHaveBeenCalledWith({
projectSlug: 'gh/org/repo',
branch: 'main',
definitionId: 'def2',
});
});
it('should run a pipeline with the first choice when only one pipeline definition is found', async () => {
vi.spyOn(projectDetection, 'getProjectSlugFromURL').mockReturnValue(
'gh/org/repo',
);
vi.spyOn(projectDetection, 'getBranchFromURL').mockReturnValue('main');
mockCircleCIClient.projects.getProject.mockResolvedValue({
id: 'project-id',
});
mockCircleCIClient.pipelines.getPipelineDefinitions.mockResolvedValue([
{ id: 'def1', name: 'Pipeline 1' },
]);
mockCircleCIClient.pipelines.runPipeline.mockResolvedValue({
number: 123,
state: 'pending',
id: 'pipeline-id',
});
const args = {
params: {
projectURL: 'https://app.circleci.com/pipelines/gh/org/repo',
},
} as any;
const controller = new AbortController();
const response = await runPipeline(args, {
signal: controller.signal,
});
expect(response).toHaveProperty('content');
expect(Array.isArray(response.content)).toBe(true);
expect(response.content[0]).toHaveProperty('type', 'text');
expect(typeof response.content[0].text).toBe('string');
expect(response.content[0].text).toContain('Pipeline run successfully');
expect(mockCircleCIClient.pipelines.runPipeline).toHaveBeenCalledWith({
projectSlug: 'gh/org/repo',
branch: 'main',
definitionId: 'def1',
});
});
it('should detect project from git remote and run pipeline', async () => {
vi.spyOn(projectDetection, 'identifyProjectSlug').mockResolvedValue(
'gh/org/repo',
);
mockCircleCIClient.projects.getProject.mockResolvedValue({
id: 'project-id',
});
mockCircleCIClient.pipelines.getPipelineDefinitions.mockResolvedValue([
{ id: 'def1', name: 'Pipeline 1' },
]);
mockCircleCIClient.pipelines.runPipeline.mockResolvedValue({
number: 123,
state: 'pending',
id: 'pipeline-id',
});
const args = {
params: {
workspaceRoot: '/workspace',
gitRemoteURL: 'https://github.com/org/repo.git',
branch: 'feature-branch',
},
} as any;
const controller = new AbortController();
const response = await runPipeline(args, {
signal: controller.signal,
});
expect(response).toHaveProperty('content');
expect(Array.isArray(response.content)).toBe(true);
expect(response.content[0]).toHaveProperty('type', 'text');
expect(typeof response.content[0].text).toBe('string');
expect(response.content[0].text).toContain('Pipeline run successfully');
expect(mockCircleCIClient.pipelines.runPipeline).toHaveBeenCalledWith({
projectSlug: 'gh/org/repo',
branch: 'feature-branch',
definitionId: 'def1',
});
});
it('should run a pipeline using projectSlug and branch inputs correctly', async () => {
mockCircleCIClient.projects.getProject.mockResolvedValue({
id: 'project-id',
});
mockCircleCIClient.pipelines.getPipelineDefinitions.mockResolvedValue([
{ id: 'def1', name: 'Pipeline 1' },
]);
mockCircleCIClient.pipelines.runPipeline.mockResolvedValue({
number: 123,
state: 'pending',
id: 'pipeline-id',
});
const args = {
params: {
projectSlug: 'gh/org/repo',
branch: 'feature/new-feature',
},
} as any;
const controller = new AbortController();
const response = await runPipeline(args, {
signal: controller.signal,
});
expect(mockCircleCIClient.projects.getProject).toHaveBeenCalledWith({
projectSlug: 'gh/org/repo',
});
expect(mockCircleCIClient.pipelines.runPipeline).toHaveBeenCalledWith({
projectSlug: 'gh/org/repo',
branch: 'feature/new-feature',
definitionId: 'def1',
});
expect(response).toHaveProperty('content');
expect(Array.isArray(response.content)).toBe(true);
expect(response.content[0]).toHaveProperty('type', 'text');
expect(typeof response.content[0].text).toBe('string');
expect(response.content[0].text).toContain('Pipeline run successfully');
});
});
```
--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
```markdown
# Changelog
All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [0.14.1] - 2025-09-17
### Fixed
- Output log messages to stderr instead of stdout, following MCP server specification. This avoids MCP server startup issues in some agents, such as Warp. Fixes #125
## [0.14.0] - 2025-08-11
### Added
- Added `download_usage_api_data` tool to start and retrieve CircleCI usage export jobs
- Added `find_underused_resource_classes` tool to analyze usage CSV for underused resource classes (default threshold 40%)
## [0.13.0] - 2025-08-05
### Added
- Added `listComponentVersions` tool to list all versions for a CircleCI component
### Changed
- Simplified `runRollbackPipeline` tool by moving part of its inner logic to `listComponentVersions`.
## [0.12.2] - 2025-08-01
### Added
- Added support for rerunning workflow in `runRollbackPipeline` when no rollback pipeline is defined.
## [0.12.1] - 2025-07-30
### Fixed
- Fixed `runRollbackPipeline` tool to not suggest other projects
## [0.12.0] - 2025-07-29
### Added
- Added `runRollbackPipeline` tool to run a rollback pipeline
## [0.11.4] - 2025-07-28
### Fixed
- Remove `tool/` prefix from tool list entries, this was breaking tool name resolution in some MCP clients.
## [0.11.3] - 2025-07-24
### Changed
- Make the diff reviewer tool work with other IDE based rule systems.
## [0.11.2] - 2025-06-20
### Added
- Add `file` property to the `RuleReviewSchema` to align with updated endpoint
## [0.11.1] - 2025-06-18
### Fixed
- Fixed bug in `get_flaky_tests` tool where unrelated tests were being returned
- Fixed bug in `get_flaky_tests` tool where if the output directory cannot be created the tool would respond with an error. We now throw in that case, which makes us fallback to the text output.
## [0.11.0] - 2025-06-18
### Fixed
- Fixed bug in `get_flaky_tests` tool where the same job number was being fetched multiple times
- Fixed bug in `get_flaky_tests` where the output directory was not being created when using file output mode
## [0.10.2] - 2025-06-18
### Added
- Add `speedMode` and `filterBy` parameters to the `analyze_diff` tool
## [0.10.1] - 2025-06-17
### Fixed
- Add a .gitignore file to the flaky-tests-output directory to ignore all files in the directory
## [0.10.0] - 2025-06-17
### Added
- Added `USE_FILE_OUTPUT` environment variable to `get_flaky_tests` tool
- When set to `true`, the tool will write flaky tests to files in the `./flaky-tests-output` directory instead of returning the results in the response
- The tool will return the file paths of the written files in the response
## [0.9.2] - 2025-06-17
### Added
- Anthropic support on prompt eval script (w. auto-detection for OpenAI and Anthropic models)
- Added `temperature` parameter support to prompt template tools
- Enhanced `create_prompt_template` tool with configurable temperature setting
- Enhanced `recommend_prompt_template_tests` tool with temperature parameter
- Default temperature value set to 1.0 for consistent prompt template generation
### Updated
- Updated default model from `gpt-4o-mini` to `gpt-4.1-mini` for prompt template tools
- Enhanced evaluation script dependencies for improved compatibility
- Updated `deepeval` to version 3.0.3+ (from 2.8.2+)
- Updated `openai` to version 1.84.0+ (from 1.76.2+)
- Added `anthropic` version 0.54.0+ for Anthropic model support
- Updated `PyYAML` to version 6.0.2+
## [0.9.1] - 2025-06-12
### Added
- Added `analyze_diff` tool to analyze git diffs against cursor rules to identify rule violations
- Evaluates code changes against repository coding standards and best practices
- Provides detailed violation reports with confidence scores and explanations
- Supports both staged and unstaged changes and all changes analysis
- Returns actionable feedback for maintaining code quality consistency
## [0.9.0] - 2025-06-03
### Added
- Added `run_evaluation_tests` tool to run evaluation tests on CircleCI pipelines
- Support for running prompt template evaluation tests in CircleCI
- Integration with prompt template files from `./prompts` directory
- Dynamic CircleCI configuration generation for evaluation workflows
- Support for multiple prompt files with automatic parallelism configuration
- Compatible with both JSON and YAML prompt template formats
- Comprehensive error handling and validation for prompt template files
- Enhanced `runPipeline` API to support custom configuration content
- Added `configContent` parameter to override default pipeline configuration
- Enables dynamic pipeline configuration for specialized use cases
## [0.8.1] - 2025-05-28
### Added
- Enhanced prompt template tools with support for existing codebase prompts
- Added `promptOrigin` parameter to distinguish between new requirements and existing codebase prompts
- Added `model` parameter to specify target model for testing (defaults to gpt-4o-mini)
- Enhanced documentation and examples for prompt template creation
- Added integration guidance for codebase-sourced prompts
- Improved prompt templates file location, naming conventions, and structure
## [0.8.0] - 2025-05-22
### Added
- Added `rerun_workflow` tool to rerun a workflow from its start or from the failed job
## [0.7.1] - 2025-05-14
### Updated
- Updated `get_build_failure_logs`, `get_job_test_results`, and `get_latest_pipeline_status` tools to require a branch parameter when using projectSlug option
## [0.7.0] - 2025-05-13
### Added
- Added `list_followed_projects` tool to list all projects that the user is following on CircleCI
## [0.6.2] - 2025-05-13
### Fixed
- Fixed `get_job_test_results` tool to filter tests by result when a job number is provided
## [0.6.1] - 2025-05-13
### Updated
- Updated `get_build_failure_logs` tool to support legacy job url format like `https://circleci.com/gh/organization/project/123`
## [0.6.0] - 2025-05-13
### Added
- Added `filterByTestsResult` parameter to `get_job_test_results` tool
- Filter the tests by result
- Support for filtering by `failure` or `success`
## [0.5.1] - 2025-05-12
### Added
- Fix handling of legacy job url format in tools
- Fix handling of pagination of test results when no test results are found
## [0.5.0] - 2025-05-09
### Added
- Added `run_pipeline` tool to run a pipeline
- Support for triggering pipelines using project URL or local git repository context
- Branch detection from URLs or local git context
- Handles multiple pipeline definitions with interactive selection
- Provides direct link to monitor pipeline execution
## [0.4.4] - 2025-05-08
### Fixed
- Fixed project detection and pipeline number extraction from URLs with custom server domains
## [0.4.3] - 2025-05-08
### Fixed
- Fixed project detection when branch is provided in URL but not in params
- Improved error handling for failed pipeline workflow fetches
- Enhanced error messaging when project is not found or inputs are missing
## [0.4.2] - 2025-05-08
### Improvements
- Enhanced prompt template file structure and organization for consistency
- Added standardized file naming convention for prompt templates
- Implemented structured JSON format with required fields (name, description, version, template, contextSchema, tests, sampleInputs, etc.)
- Added support for test case naming in Title Case format
- Improved documentation requirements for prompt templates
## [0.4.1] - 2025-05-05
### Added
- Update project detection to correctly paginate the followed projects
## [0.4.0] - 2025-04-30
### Added
- Added `get_job_test_results` tool to retrieve and analyze test metadata from CircleCI jobs
- Support for retrieving test results using job, workflow, or pipeline URLs
- Support for retrieving test results using local git repository context
- Displays comprehensive test result summary (total, successful, failed)
- Provides detailed information for failed tests including name, class, file, error messages, and runtime
- Lists successful tests with timing information
- Offers actionable guidance when no test results are found
- Includes documentation link to help users properly configure test metadata collection
## [0.3.0] - 2025-04-30
### Added
- Added `get_latest_pipeline_status` tool to get the latest pipeline status
- Support for both project URL and local git repository context
- Displays all workflows within the latest pipeline
- Provides formatted details including pipeline number, workflow status, duration, and timestamps
## [0.2.0] - 2025-04-18
### Added
- Added `create_prompt_template` tool to help generate structured prompt templates
- Converts feature requirements into optimized prompt templates
- Generates context schema for input parameters
- Enables building robust AI-powered features
- Integrates with prompt template testing workflow
- Added `recommend_prompt_template_tests` tool for prompt template validation
- Creates diverse test scenarios based on templates
- Generates test cases with varied parameter combinations
- Helps identify edge cases and potential issues
- Ensures consistent AI responses across inputs
## [0.1.10] - 2025-04-17
### Fixed
- Fixed rate limiting issues when fetching job logs and flaky tests (#32)
- Implemented `rateLimitedRequests` utility for controlled API request batching
- Added configurable batch size and interval controls
- Improved error handling for rate-limited responses
- Added progress tracking for batch operations
- Applied rate limiting fix to both job logs and flaky test detection
- Enhanced reliability of test results retrieval
### Improvements
- Enhanced HTTP client configuration flexibility
- Configurable base URL through environment variables
- Better support for different CircleCI deployment scenarios
- Streamlined client initialization process
- Added output text truncation
- Prevents response overload by limiting output size
- Includes clear warning when content is truncated
- Preserves most recent and relevant information
## [0.1.9] - 2025-04-16
### Added
- Added support for API subdomain configuration in CircleCI client
- New `useAPISubdomain` option in HTTP client configuration
- Automatic subdomain handling for API-specific endpoints
- Improved support for CircleCI enterprise and on-premise installations
- Added `config_helper` tool to assist with CircleCI configuration tasks
- Support for validating .circleci/config.yml files
- Integration with CircleCI Config Validation API
- Detailed validation results and configuration recommendations
- Helpful error messages and best practice suggestions
## [0.1.8] - 2025-04-10
### Fixed
- Fixed bug in flaky test detection where pipelineNumber was incorrectly used instead of projectSlug when URL not provided
### Improvements
- Consolidated project slug detection functions into a single `getPipelineNumberFromURL` function with enhanced test coverage
- Simplified build logs tool to use only `projectURL` parameter instead of separate pipeline and job URLs
- Updated tool descriptions to provide clearer guidance on accepted URL formats
- Removed redundant error handling wrapper
## [0.1.7] - 2025-04-10
### Added
- Added `find_flaky_tests` tool to identify and analyze flaky tests in CircleCI projects
- Support for both project URL and local git repository context
- Integration with CircleCI Insights API for flaky test detection
- Integration with CircleCI Tests API to fetch detailed test execution results
- Formatted output of flaky test analysis results with complete test logs
## [0.1.6] - 2025-04-09
### Added
- Added User-Agent header to CircleCI API requests
## [0.1.5] - 2025-04-08
### Added
- Support for configurable CircleCI base URL through `CIRCLECI_BASE_URL` environment variable
## [0.1.4] - 2025-04-08
### Fixed
- Handle missing job numbers in CircleCI API responses by making job_number optional in schema
- Skip jobs without job numbers when fetching job logs instead of failing
## [0.1.3] - 2025-04-04
### Added
- Improved schema validation and output formatting for job information
## [0.1.2] - 2025-04-04
### Fixed
- More permissive schema validation for CircleCI API parameters
- Allow optional parameters in API requests
### Documentation
- Updated documentation around package publishing
- Removed note about package not being published
## [0.1.1] - 2025-04-04
### Fixed
- Non functional fixes
## [0.1.0] - 2025-04-04
Initial release of the CircleCI MCP Server, enabling natural language interactions with CircleCI functionality through MCP-enabled clients.
### Added
- Core MCP server implementation with CircleCI integration
- Support for MCP protocol version 1.8.0
- Robust error handling and response formatting
- Standardized HTTP client for CircleCI API interactions
- CircleCI API Integration
- Support for both CircleCI API v1.1 and v2
- Comprehensive API client implementation for Jobs, Pipelines, and Workflows
- Private API integration for enhanced functionality
- Secure token-based authentication
- Build Failure Analysis Tool
- Implemented `get_build_failure_logs` tool for retrieving detailed failure logs
- Support for both URL-based and local project context-based queries
- Intelligent project detection from git repository information
- Formatted log output with job names and step-by-step execution details
- Development Tools and Infrastructure
- Comprehensive test suite with Vitest
- ESLint and Prettier configuration for code quality
- TypeScript configuration for type safety
- Development workflow with MCP Inspector support
- Watch mode for rapid development
### Security
- Secure handling of CircleCI API tokens
- Masked sensitive data in log outputs
- Proper error handling to prevent information leakage
```
--------------------------------------------------------------------------------
/src/tools/runEvaluationTests/handler.test.ts:
--------------------------------------------------------------------------------
```typescript
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { runEvaluationTests } from './handler.js';
import * as projectDetection from '../../lib/project-detection/index.js';
import * as clientModule from '../../clients/client.js';
vi.mock('../../lib/project-detection/index.js');
vi.mock('../../clients/client.js');
describe('runEvaluationTests handler', () => {
const mockCircleCIClient = {
projects: {
getProject: vi.fn(),
},
pipelines: {
getPipelineDefinitions: vi.fn(),
runPipeline: vi.fn(),
},
};
beforeEach(() => {
vi.resetAllMocks();
vi.spyOn(clientModule, 'getCircleCIClient').mockReturnValue(
mockCircleCIClient as any,
);
});
it('should return a valid MCP error response when no inputs are provided', async () => {
const args = {
params: {},
} as any;
const controller = new AbortController();
const response = await runEvaluationTests(args, {
signal: controller.signal,
});
expect(response).toHaveProperty('content');
expect(response).toHaveProperty('isError', true);
expect(Array.isArray(response.content)).toBe(true);
expect(response.content[0]).toHaveProperty('type', 'text');
expect(typeof response.content[0].text).toBe('string');
});
it('should return a valid MCP error response when project is not found', async () => {
vi.spyOn(projectDetection, 'identifyProjectSlug').mockResolvedValue(
undefined,
);
const args = {
params: {
workspaceRoot: '/workspace',
gitRemoteURL: 'https://github.com/org/repo.git',
branch: 'main',
promptFiles: [
{
fileName: 'test.prompt.yml',
fileContent: 'test: content',
},
],
},
} as any;
const controller = new AbortController();
const response = await runEvaluationTests(args, {
signal: controller.signal,
});
expect(response).toHaveProperty('content');
expect(response).toHaveProperty('isError', true);
expect(Array.isArray(response.content)).toBe(true);
expect(response.content[0]).toHaveProperty('type', 'text');
expect(typeof response.content[0].text).toBe('string');
});
it('should return a valid MCP error response when no branch is provided', async () => {
vi.spyOn(projectDetection, 'getProjectSlugFromURL').mockReturnValue(
'gh/org/repo',
);
vi.spyOn(projectDetection, 'getBranchFromURL').mockReturnValue(undefined);
const args = {
params: {
projectURL: 'https://app.circleci.com/pipelines/gh/org/repo',
promptFiles: [
{
fileName: 'test.prompt.yml',
fileContent: 'test: content',
},
],
},
} as any;
const controller = new AbortController();
const response = await runEvaluationTests(args, {
signal: controller.signal,
});
expect(response).toHaveProperty('content');
expect(response).toHaveProperty('isError', true);
expect(Array.isArray(response.content)).toBe(true);
expect(response.content[0]).toHaveProperty('type', 'text');
expect(typeof response.content[0].text).toBe('string');
expect(response.content[0].text).toContain('No branch provided');
});
it('should return a valid MCP error response when projectSlug is provided without branch', async () => {
const args = {
params: {
projectSlug: 'gh/org/repo',
promptFiles: [
{
fileName: 'test.prompt.yml',
fileContent: 'test: content',
},
],
// No branch provided
},
} as any;
const controller = new AbortController();
const response = await runEvaluationTests(args, {
signal: controller.signal,
});
expect(response).toHaveProperty('content');
expect(response).toHaveProperty('isError', true);
expect(Array.isArray(response.content)).toBe(true);
expect(response.content[0]).toHaveProperty('type', 'text');
expect(response.content[0].text).toContain('Branch not provided');
// Verify that CircleCI API was not called
expect(mockCircleCIClient.projects.getProject).not.toHaveBeenCalled();
expect(
mockCircleCIClient.pipelines.getPipelineDefinitions,
).not.toHaveBeenCalled();
expect(mockCircleCIClient.pipelines.runPipeline).not.toHaveBeenCalled();
});
it('should return a valid MCP error response when no prompt files are provided', async () => {
const args = {
params: {
projectSlug: 'gh/org/repo',
branch: 'main',
promptFiles: [], // Empty array
},
} as any;
const controller = new AbortController();
const response = await runEvaluationTests(args, {
signal: controller.signal,
});
expect(response).toHaveProperty('content');
expect(response).toHaveProperty('isError', true);
expect(Array.isArray(response.content)).toBe(true);
expect(response.content[0]).toHaveProperty('type', 'text');
expect(response.content[0].text).toContain(
'No prompt template files provided',
);
expect(response.content[0].text).toContain('./prompts directory');
// Verify that CircleCI API was not called
expect(mockCircleCIClient.projects.getProject).not.toHaveBeenCalled();
expect(
mockCircleCIClient.pipelines.getPipelineDefinitions,
).not.toHaveBeenCalled();
expect(mockCircleCIClient.pipelines.runPipeline).not.toHaveBeenCalled();
});
it('should return a valid MCP error response when promptFiles is undefined', async () => {
const args = {
params: {
projectSlug: 'gh/org/repo',
branch: 'main',
// promptFiles is undefined
},
} as any;
const controller = new AbortController();
const response = await runEvaluationTests(args, {
signal: controller.signal,
});
expect(response).toHaveProperty('content');
expect(response).toHaveProperty('isError', true);
expect(Array.isArray(response.content)).toBe(true);
expect(response.content[0]).toHaveProperty('type', 'text');
expect(response.content[0].text).toContain(
'No prompt template files provided',
);
expect(response.content[0].text).toContain('./prompts directory');
// Verify that CircleCI API was not called
expect(mockCircleCIClient.projects.getProject).not.toHaveBeenCalled();
expect(
mockCircleCIClient.pipelines.getPipelineDefinitions,
).not.toHaveBeenCalled();
expect(mockCircleCIClient.pipelines.runPipeline).not.toHaveBeenCalled();
});
it('should return a valid MCP error response when no pipeline definitions are found', async () => {
vi.spyOn(projectDetection, 'getProjectSlugFromURL').mockReturnValue(
'gh/org/repo',
);
vi.spyOn(projectDetection, 'getBranchFromURL').mockReturnValue('main');
mockCircleCIClient.projects.getProject.mockResolvedValue({
id: 'project-id',
});
mockCircleCIClient.pipelines.getPipelineDefinitions.mockResolvedValue([]);
const args = {
params: {
projectURL: 'https://app.circleci.com/pipelines/gh/org/repo',
promptFiles: [
{
fileName: 'test.prompt.yml',
fileContent: 'test: content',
},
],
},
} as any;
const controller = new AbortController();
const response = await runEvaluationTests(args, {
signal: controller.signal,
});
expect(response).toHaveProperty('content');
expect(response).toHaveProperty('isError', true);
expect(Array.isArray(response.content)).toBe(true);
expect(response.content[0]).toHaveProperty('type', 'text');
expect(typeof response.content[0].text).toBe('string');
expect(response.content[0].text).toContain('No pipeline definitions found');
});
it('should return a list of pipeline choices when multiple pipeline definitions are found and no choice is provided', async () => {
vi.spyOn(projectDetection, 'getProjectSlugFromURL').mockReturnValue(
'gh/org/repo',
);
vi.spyOn(projectDetection, 'getBranchFromURL').mockReturnValue('main');
mockCircleCIClient.projects.getProject.mockResolvedValue({
id: 'project-id',
});
mockCircleCIClient.pipelines.getPipelineDefinitions.mockResolvedValue([
{ id: 'def1', name: 'Pipeline 1' },
{ id: 'def2', name: 'Pipeline 2' },
]);
const args = {
params: {
projectURL: 'https://app.circleci.com/pipelines/gh/org/repo',
promptFiles: [
{
fileName: 'test.prompt.yml',
fileContent: 'test: content',
},
],
},
} as any;
const controller = new AbortController();
const response = await runEvaluationTests(args, {
signal: controller.signal,
});
expect(response).toHaveProperty('content');
expect(Array.isArray(response.content)).toBe(true);
expect(response.content[0]).toHaveProperty('type', 'text');
expect(typeof response.content[0].text).toBe('string');
expect(response.content[0].text).toContain(
'Multiple pipeline definitions found',
);
expect(response.content[0].text).toContain('Pipeline 1');
expect(response.content[0].text).toContain('Pipeline 2');
});
it('should return an error when an invalid pipeline choice is provided', async () => {
vi.spyOn(projectDetection, 'getProjectSlugFromURL').mockReturnValue(
'gh/org/repo',
);
vi.spyOn(projectDetection, 'getBranchFromURL').mockReturnValue('main');
mockCircleCIClient.projects.getProject.mockResolvedValue({
id: 'project-id',
});
mockCircleCIClient.pipelines.getPipelineDefinitions.mockResolvedValue([
{ id: 'def1', name: 'Pipeline 1' },
{ id: 'def2', name: 'Pipeline 2' },
]);
const args = {
params: {
projectURL: 'https://app.circleci.com/pipelines/gh/org/repo',
pipelineChoiceName: 'Non-existent Pipeline',
promptFiles: [
{
fileName: 'test.prompt.yml',
fileContent: 'test: content',
},
],
},
} as any;
const controller = new AbortController();
const response = await runEvaluationTests(args, {
signal: controller.signal,
});
expect(response).toHaveProperty('content');
expect(response).toHaveProperty('isError', true);
expect(Array.isArray(response.content)).toBe(true);
expect(response.content[0]).toHaveProperty('type', 'text');
expect(typeof response.content[0].text).toBe('string');
expect(response.content[0].text).toContain(
'Pipeline definition with name Non-existent Pipeline not found',
);
});
it('should run evaluation tests with multiple prompt files and correct parallelism', async () => {
vi.spyOn(projectDetection, 'getProjectSlugFromURL').mockReturnValue(
'gh/org/repo',
);
vi.spyOn(projectDetection, 'getBranchFromURL').mockReturnValue('main');
mockCircleCIClient.projects.getProject.mockResolvedValue({
id: 'project-id',
});
mockCircleCIClient.pipelines.getPipelineDefinitions.mockResolvedValue([
{ id: 'def1', name: 'Pipeline 1' },
]);
mockCircleCIClient.pipelines.runPipeline.mockResolvedValue({
number: 123,
state: 'pending',
id: 'pipeline-id',
});
const args = {
params: {
projectURL: 'https://app.circleci.com/pipelines/gh/org/repo',
promptFiles: [
{
fileName: 'test1.prompt.json',
fileContent: '{"template": "test content 1"}',
},
{
fileName: 'test2.prompt.yml',
fileContent: 'template: test content 2',
},
],
},
} as any;
const controller = new AbortController();
const response = await runEvaluationTests(args, {
signal: controller.signal,
});
expect(response).toHaveProperty('content');
expect(Array.isArray(response.content)).toBe(true);
expect(response.content[0]).toHaveProperty('type', 'text');
expect(typeof response.content[0].text).toBe('string');
expect(response.content[0].text).toContain('Pipeline run successfully');
// Verify that the pipeline was called with correct configuration
expect(mockCircleCIClient.pipelines.runPipeline).toHaveBeenCalledWith({
projectSlug: 'gh/org/repo',
branch: 'main',
definitionId: 'def1',
configContent: expect.stringContaining('parallelism: 2'), // Should match number of files
});
// Verify the config contains conditional file creation logic
const configContent =
mockCircleCIClient.pipelines.runPipeline.mock.calls[0][0].configContent;
expect(configContent).toContain('CIRCLE_NODE_INDEX');
expect(configContent).toContain('test1.prompt.json');
expect(configContent).toContain('test2.prompt.yml');
expect(configContent).toContain('python eval.py');
});
it('should process JSON files with proper formatting', async () => {
vi.spyOn(projectDetection, 'getProjectSlugFromURL').mockReturnValue(
'gh/org/repo',
);
vi.spyOn(projectDetection, 'getBranchFromURL').mockReturnValue('main');
mockCircleCIClient.projects.getProject.mockResolvedValue({
id: 'project-id',
});
mockCircleCIClient.pipelines.getPipelineDefinitions.mockResolvedValue([
{ id: 'def1', name: 'Pipeline 1' },
]);
mockCircleCIClient.pipelines.runPipeline.mockResolvedValue({
number: 123,
state: 'pending',
id: 'pipeline-id',
});
const args = {
params: {
projectSlug: 'gh/org/repo',
branch: 'main',
promptFiles: [
{
fileName: 'test.prompt.json',
fileContent: '{"template":"test","vars":["a","b"]}',
},
],
},
} as any;
const controller = new AbortController();
await runEvaluationTests(args, {
signal: controller.signal,
});
// Verify that the pipeline was called
expect(mockCircleCIClient.pipelines.runPipeline).toHaveBeenCalled();
const configContent =
mockCircleCIClient.pipelines.runPipeline.mock.calls[0][0].configContent;
expect(configContent).toContain('parallelism: 1');
expect(configContent).toContain('test.prompt.json');
});
it('should detect project from git remote and run evaluation tests', async () => {
vi.spyOn(projectDetection, 'identifyProjectSlug').mockResolvedValue(
'gh/org/repo',
);
mockCircleCIClient.projects.getProject.mockResolvedValue({
id: 'project-id',
});
mockCircleCIClient.pipelines.getPipelineDefinitions.mockResolvedValue([
{ id: 'def1', name: 'Pipeline 1' },
]);
mockCircleCIClient.pipelines.runPipeline.mockResolvedValue({
number: 123,
state: 'pending',
id: 'pipeline-id',
});
const args = {
params: {
workspaceRoot: '/workspace',
gitRemoteURL: 'https://github.com/org/repo.git',
branch: 'feature-branch',
promptFiles: [
{
fileName: 'test.prompt.yml',
fileContent: 'template: test content',
},
],
},
} as any;
const controller = new AbortController();
const response = await runEvaluationTests(args, {
signal: controller.signal,
});
expect(response).toHaveProperty('content');
expect(Array.isArray(response.content)).toBe(true);
expect(response.content[0]).toHaveProperty('type', 'text');
expect(typeof response.content[0].text).toBe('string');
expect(response.content[0].text).toContain('Pipeline run successfully');
expect(mockCircleCIClient.pipelines.runPipeline).toHaveBeenCalledWith({
projectSlug: 'gh/org/repo',
branch: 'feature-branch',
definitionId: 'def1',
configContent: expect.any(String),
});
});
});
```
--------------------------------------------------------------------------------
/src/tools/listComponentVersions/handler.test.ts:
--------------------------------------------------------------------------------
```typescript
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { listComponentVersions } from './handler.js';
import * as clientModule from '../../clients/client.js';
vi.mock('../../clients/client.js');
describe('listComponentVersions handler', () => {
const mockCircleCIClient = {
deploys: {
fetchComponentVersions: vi.fn(),
fetchEnvironments: vi.fn(),
fetchProjectComponents: vi.fn(),
},
projects: {
getProject: vi.fn(),
getProjectByID: vi.fn(),
},
};
const mockExtra = {
signal: new AbortController().signal,
requestId: 'test-id',
sendNotification: vi.fn(),
sendRequest: vi.fn(),
};
beforeEach(() => {
vi.resetAllMocks();
vi.spyOn(clientModule, 'getCircleCIClient').mockReturnValue(
mockCircleCIClient as any,
);
});
it('should return the formatted component versions when found', async () => {
const mockComponentVersions = {
items: [
{
id: 'version-1',
component_id: 'test-component-id',
environment_id: 'test-environment-id',
version: '1.0.0',
sha: 'abc123',
created_at: '2023-01-01T00:00:00Z',
updated_at: '2023-01-02T00:00:00Z',
is_live: true,
},
{
id: 'version-2',
component_id: 'test-component-id',
environment_id: 'test-environment-id',
version: '1.1.0',
sha: 'def456',
created_at: '2023-01-03T00:00:00Z',
updated_at: '2023-01-04T00:00:00Z',
is_live: false,
},
],
next_page_token: null,
};
// Mock project resolution
mockCircleCIClient.projects.getProject.mockResolvedValue({
id: 'test-project-id',
organization_id: 'test-org-id',
});
mockCircleCIClient.deploys.fetchComponentVersions.mockResolvedValue(mockComponentVersions);
const args = {
params: {
projectSlug: 'gh/test-org/test-repo',
componentID: 'test-component-id',
environmentID: 'test-environment-id',
},
} as any;
const response = await listComponentVersions(args, mockExtra);
expect(response).toHaveProperty('content');
expect(Array.isArray(response.content)).toBe(true);
expect(response.content[0]).toHaveProperty('type', 'text');
expect(typeof response.content[0].text).toBe('string');
expect(response.content[0].text).toContain('Versions for the component:');
expect(response.content[0].text).toContain(JSON.stringify(mockComponentVersions));
expect(mockCircleCIClient.projects.getProject).toHaveBeenCalledTimes(1);
expect(mockCircleCIClient.projects.getProject).toHaveBeenCalledWith({
projectSlug: 'gh/test-org/test-repo',
});
expect(mockCircleCIClient.deploys.fetchComponentVersions).toHaveBeenCalledTimes(1);
expect(mockCircleCIClient.deploys.fetchComponentVersions).toHaveBeenCalledWith({
componentID: 'test-component-id',
environmentID: 'test-environment-id',
});
});
it('should return "No component versions found" when component versions list is empty', async () => {
// Mock project resolution
mockCircleCIClient.projects.getProject.mockResolvedValue({
id: 'test-project-id',
organization_id: 'test-org-id',
});
mockCircleCIClient.deploys.fetchComponentVersions.mockResolvedValue({
items: [],
next_page_token: null,
});
const args = {
params: {
projectSlug: 'gh/test-org/test-repo',
componentID: 'test-component-id',
environmentID: 'test-environment-id',
},
} as any;
const response = await listComponentVersions(args, mockExtra);
expect(response).toHaveProperty('content');
expect(Array.isArray(response.content)).toBe(true);
expect(response.content[0]).toHaveProperty('type', 'text');
expect(typeof response.content[0].text).toBe('string');
expect(response.content[0].text).toBe('No component versions found');
});
it('should handle API errors gracefully', async () => {
const errorMessage = 'Component versions API request failed';
// Mock project resolution
mockCircleCIClient.projects.getProject.mockResolvedValue({
id: 'test-project-id',
organization_id: 'test-org-id',
});
mockCircleCIClient.deploys.fetchComponentVersions.mockRejectedValue(
new Error(errorMessage),
);
const args = {
params: {
projectSlug: 'gh/test-org/test-repo',
componentID: 'test-component-id',
environmentID: 'test-environment-id',
},
} as any;
const response = await listComponentVersions(args, mockExtra);
expect(response).toHaveProperty('content');
expect(response).toHaveProperty('isError', true);
expect(Array.isArray(response.content)).toBe(true);
expect(response.content[0]).toHaveProperty('type', 'text');
expect(typeof response.content[0].text).toBe('string');
expect(response.content[0].text).toContain('Failed to list component versions:');
expect(response.content[0].text).toContain(errorMessage);
});
it('should handle non-Error exceptions gracefully', async () => {
// Mock project resolution
mockCircleCIClient.projects.getProject.mockResolvedValue({
id: 'test-project-id',
organization_id: 'test-org-id',
});
mockCircleCIClient.deploys.fetchComponentVersions.mockRejectedValue(
'Unexpected error',
);
const args = {
params: {
projectSlug: 'gh/test-org/test-repo',
componentID: 'test-component-id',
environmentID: 'test-environment-id',
},
} as any;
const response = await listComponentVersions(args, mockExtra);
expect(response).toHaveProperty('content');
expect(response).toHaveProperty('isError', true);
expect(Array.isArray(response.content)).toBe(true);
expect(response.content[0]).toHaveProperty('type', 'text');
expect(typeof response.content[0].text).toBe('string');
expect(response.content[0].text).toContain('Failed to list component versions:');
expect(response.content[0].text).toContain('Unknown error');
});
describe('Project resolution scenarios', () => {
it('should use projectID and orgID directly when both are provided', async () => {
const mockComponentVersions = {
items: [
{
id: 'version-1',
component_id: 'test-component-id',
environment_id: 'test-environment-id',
version: '1.0.0',
sha: 'abc123',
created_at: '2023-01-01T00:00:00Z',
updated_at: '2023-01-02T00:00:00Z',
is_live: true,
},
],
next_page_token: null,
};
mockCircleCIClient.deploys.fetchComponentVersions.mockResolvedValue(mockComponentVersions);
const args = {
params: {
projectID: 'test-project-id',
orgID: 'test-org-id',
componentID: 'test-component-id',
environmentID: 'test-environment-id',
},
} as any;
const response = await listComponentVersions(args, mockExtra);
expect(response).toHaveProperty('content');
expect(response.content[0].text).toContain('Versions for the component:');
// Should not call project resolution methods when both IDs are provided
expect(mockCircleCIClient.projects.getProject).not.toHaveBeenCalled();
expect(mockCircleCIClient.projects.getProjectByID).not.toHaveBeenCalled();
expect(mockCircleCIClient.deploys.fetchComponentVersions).toHaveBeenCalledWith({
componentID: 'test-component-id',
environmentID: 'test-environment-id',
});
});
it('should resolve orgID from projectID when only projectID is provided', async () => {
const mockComponentVersions = {
items: [
{
id: 'version-1',
component_id: 'test-component-id',
environment_id: 'test-environment-id',
version: '1.0.0',
sha: 'abc123',
created_at: '2023-01-01T00:00:00Z',
updated_at: '2023-01-02T00:00:00Z',
is_live: true,
},
],
next_page_token: null,
};
mockCircleCIClient.projects.getProjectByID.mockResolvedValue({
id: 'test-project-id',
organization_id: 'resolved-org-id',
});
mockCircleCIClient.deploys.fetchComponentVersions.mockResolvedValue(mockComponentVersions);
const args = {
params: {
projectID: 'test-project-id',
componentID: 'test-component-id',
environmentID: 'test-environment-id',
},
} as any;
const response = await listComponentVersions(args, mockExtra);
expect(response).toHaveProperty('content');
expect(response.content[0].text).toContain('Versions for the component:');
expect(mockCircleCIClient.projects.getProjectByID).toHaveBeenCalledWith({
projectID: 'test-project-id',
});
expect(mockCircleCIClient.projects.getProject).not.toHaveBeenCalled();
});
it('should handle project resolution errors for projectSlug', async () => {
const errorMessage = 'Project not found';
mockCircleCIClient.projects.getProject.mockRejectedValue(
new Error(errorMessage),
);
const args = {
params: {
projectSlug: 'gh/invalid/repo',
componentID: 'test-component-id',
environmentID: 'test-environment-id',
},
} as any;
const response = await listComponentVersions(args, mockExtra);
expect(response).toHaveProperty('content');
expect(response).toHaveProperty('isError', true);
expect(response.content[0].text).toContain('Failed to resolve project information for gh/invalid/repo');
expect(response.content[0].text).toContain(errorMessage);
});
it('should handle project resolution errors for projectID', async () => {
const errorMessage = 'Project ID not found';
mockCircleCIClient.projects.getProjectByID.mockRejectedValue(
new Error(errorMessage),
);
const args = {
params: {
projectID: 'invalid-project-id',
componentID: 'test-component-id',
environmentID: 'test-environment-id',
},
} as any;
const response = await listComponentVersions(args, mockExtra);
expect(response).toHaveProperty('content');
expect(response).toHaveProperty('isError', true);
expect(response.content[0].text).toContain('Failed to resolve project information for project ID invalid-project-id');
expect(response.content[0].text).toContain(errorMessage);
});
it('should return error when neither projectSlug nor projectID is provided', async () => {
const args = {
params: {
componentID: 'test-component-id',
environmentID: 'test-environment-id',
},
} as any;
const response = await listComponentVersions(args, mockExtra);
expect(response).toHaveProperty('content');
expect(response).toHaveProperty('isError', true);
expect(response.content[0].text).toContain('Invalid request. Please specify either a project slug or a project ID.');
});
});
describe('Missing environmentID scenarios', () => {
it('should list environments when environmentID is not provided', async () => {
const mockEnvironments = {
items: [
{ id: 'env-1', name: 'production' },
{ id: 'env-2', name: 'staging' },
],
next_page_token: null,
};
mockCircleCIClient.projects.getProject.mockResolvedValue({
id: 'test-project-id',
organization_id: 'test-org-id',
});
mockCircleCIClient.deploys.fetchEnvironments.mockResolvedValue(mockEnvironments);
const args = {
params: {
projectSlug: 'gh/test-org/test-repo',
componentID: 'test-component-id',
},
} as any;
const response = await listComponentVersions(args, mockExtra);
expect(response).toHaveProperty('content');
expect(response.content[0].text).toContain('Please provide an environmentID. Available environments:');
expect(response.content[0].text).toContain('1. production (ID: env-1)');
expect(response.content[0].text).toContain('2. staging (ID: env-2)');
expect(mockCircleCIClient.deploys.fetchEnvironments).toHaveBeenCalledWith({
orgID: 'test-org-id',
});
});
it('should handle empty environments list', async () => {
mockCircleCIClient.projects.getProject.mockResolvedValue({
id: 'test-project-id',
organization_id: 'test-org-id',
});
mockCircleCIClient.deploys.fetchEnvironments.mockResolvedValue({
items: [],
next_page_token: null,
});
const args = {
params: {
projectSlug: 'gh/test-org/test-repo',
componentID: 'test-component-id',
},
} as any;
const response = await listComponentVersions(args, mockExtra);
expect(response).toHaveProperty('content');
expect(response.content[0].text).toBe('No environments found');
});
it('should handle fetchEnvironments API errors', async () => {
const errorMessage = 'Failed to fetch environments';
mockCircleCIClient.projects.getProject.mockResolvedValue({
id: 'test-project-id',
organization_id: 'test-org-id',
});
mockCircleCIClient.deploys.fetchEnvironments.mockRejectedValue(
new Error(errorMessage),
);
const args = {
params: {
projectSlug: 'gh/test-org/test-repo',
componentID: 'test-component-id',
},
} as any;
const response = await listComponentVersions(args, mockExtra);
expect(response).toHaveProperty('content');
expect(response).toHaveProperty('isError', true);
expect(response.content[0].text).toContain('Failed to list component versions:');
expect(response.content[0].text).toContain(errorMessage);
});
});
describe('Missing componentID scenarios', () => {
it('should list components when componentID is not provided', async () => {
const mockComponents = {
items: [
{ id: 'comp-1', name: 'frontend' },
{ id: 'comp-2', name: 'backend' },
],
next_page_token: null,
};
mockCircleCIClient.projects.getProject.mockResolvedValue({
id: 'test-project-id',
organization_id: 'test-org-id',
});
mockCircleCIClient.deploys.fetchProjectComponents.mockResolvedValue(mockComponents);
const args = {
params: {
projectSlug: 'gh/test-org/test-repo',
environmentID: 'test-environment-id',
},
} as any;
const response = await listComponentVersions(args, mockExtra);
expect(response).toHaveProperty('content');
expect(response.content[0].text).toContain('Please provide a componentID. Available components:');
expect(response.content[0].text).toContain('1. frontend (ID: comp-1)');
expect(response.content[0].text).toContain('2. backend (ID: comp-2)');
expect(mockCircleCIClient.deploys.fetchProjectComponents).toHaveBeenCalledWith({
projectID: 'test-project-id',
orgID: 'test-org-id',
});
});
it('should handle empty components list', async () => {
mockCircleCIClient.projects.getProject.mockResolvedValue({
id: 'test-project-id',
organization_id: 'test-org-id',
});
mockCircleCIClient.deploys.fetchProjectComponents.mockResolvedValue({
items: [],
next_page_token: null,
});
const args = {
params: {
projectSlug: 'gh/test-org/test-repo',
environmentID: 'test-environment-id',
},
} as any;
const response = await listComponentVersions(args, mockExtra);
expect(response).toHaveProperty('content');
expect(response.content[0].text).toBe('No components found');
});
it('should handle fetchProjectComponents API errors', async () => {
const errorMessage = 'Failed to fetch components';
mockCircleCIClient.projects.getProject.mockResolvedValue({
id: 'test-project-id',
organization_id: 'test-org-id',
});
mockCircleCIClient.deploys.fetchProjectComponents.mockRejectedValue(
new Error(errorMessage),
);
const args = {
params: {
projectSlug: 'gh/test-org/test-repo',
environmentID: 'test-environment-id',
},
} as any;
const response = await listComponentVersions(args, mockExtra);
expect(response).toHaveProperty('content');
expect(response).toHaveProperty('isError', true);
expect(response.content[0].text).toContain('Failed to list component versions:');
expect(response.content[0].text).toContain(errorMessage);
});
});
});
```