This is page 2 of 4. Use http://codebase.md/marianfoo/mcp-sap-docs?page={x} to view the full context.
# Directory Structure
```
├── .cursor
│ └── rules
│ ├── 00-overview.mdc
│ ├── 10-search-stack.mdc
│ ├── 20-tools-and-apis.mdc
│ ├── 30-tests-and-output.mdc
│ ├── 40-deploy.mdc
│ ├── 50-metadata-config.mdc
│ ├── 60-adding-github-sources.mdc
│ ├── 70-tool-usage-guide.mdc
│ └── 80-abap-integration.mdc
├── .cursorignore
├── .gitattributes
├── .github
│ ├── ISSUE_TEMPLATE
│ │ ├── config.yml
│ │ ├── missing-documentation.yml
│ │ └── new-documentation-source.yml
│ └── workflows
│ ├── deploy-mcp-sap-docs.yml
│ ├── test-pr.yml
│ └── update-submodules.yml
├── .gitignore
├── .gitmodules
├── .npmignore
├── .vscode
│ ├── extensions.json
│ └── settings.json
├── docs
│ ├── ABAP-INTEGRATION-SUMMARY.md
│ ├── ABAP-MULTI-VERSION-INTEGRATION.md
│ ├── ABAP-STANDARD-INTEGRATION.md
│ ├── ABAP-USAGE-GUIDE.md
│ ├── ARCHITECTURE.md
│ ├── COMMUNITY-SEARCH-IMPLEMENTATION.md
│ ├── CONTENT-SIZE-LIMITS.md
│ ├── CURSOR-SETUP.md
│ ├── DEV.md
│ ├── FTS5-IMPLEMENTATION-COMPLETE.md
│ ├── LLM-FRIENDLY-IMPROVEMENTS.md
│ ├── METADATA-CONSOLIDATION.md
│ ├── TEST-SEARCH.md
│ └── TESTS.md
├── ecosystem.config.cjs
├── index.html
├── LICENSE
├── package-lock.json
├── package.json
├── README.md
├── REMOTE_SETUP.md
├── scripts
│ ├── build-fts.ts
│ ├── build-index.ts
│ ├── check-version.js
│ └── summarize-src.js
├── server.json
├── setup.sh
├── src
│ ├── global.d.ts
│ ├── http-server.ts
│ ├── lib
│ │ ├── BaseServerHandler.ts
│ │ ├── communityBestMatch.ts
│ │ ├── config.ts
│ │ ├── localDocs.ts
│ │ ├── logger.ts
│ │ ├── metadata.ts
│ │ ├── sapHelp.ts
│ │ ├── search.ts
│ │ ├── searchDb.ts
│ │ ├── truncate.ts
│ │ ├── types.ts
│ │ └── url-generation
│ │ ├── abap.ts
│ │ ├── BaseUrlGenerator.ts
│ │ ├── cap.ts
│ │ ├── cloud-sdk.ts
│ │ ├── dsag.ts
│ │ ├── GenericUrlGenerator.ts
│ │ ├── index.ts
│ │ ├── README.md
│ │ ├── sapui5.ts
│ │ ├── utils.ts
│ │ └── wdi5.ts
│ ├── metadata.json
│ ├── server.ts
│ └── streamable-http-server.ts
├── test
│ ├── _utils
│ │ ├── httpClient.js
│ │ └── parseResults.js
│ ├── community-search.ts
│ ├── comprehensive-url-generation.test.ts
│ ├── performance
│ │ └── README.md
│ ├── prompts.test.ts
│ ├── quick-url-test.ts
│ ├── README.md
│ ├── tools
│ │ ├── run-tests.js
│ │ ├── sap_docs_search
│ │ │ ├── search-cap-docs.js
│ │ │ ├── search-cloud-sdk-ai.js
│ │ │ ├── search-cloud-sdk-js.js
│ │ │ └── search-sapui5-docs.js
│ │ ├── search-url-verification.js
│ │ ├── search.generic.spec.js
│ │ └── search.smoke.js
│ ├── url-status.ts
│ └── validate-urls.ts
├── test-community-search.js
├── test-search-interactive.ts
├── test-search.http
├── test-search.ts
├── tsconfig.json
└── vitest.config.ts
```
# Files
--------------------------------------------------------------------------------
/docs/ABAP-MULTI-VERSION-INTEGRATION.md:
--------------------------------------------------------------------------------
```markdown
# ✅ **ABAP Multi-Version Integration Complete**
## 🎯 **Integration Summary**
ABAP documentation is now fully integrated as **standard sources** across all versions with intelligent auto-detection capabilities.
### **📊 Statistics: 42,901 ABAP Files Across 8 Versions**
| Version | Files | Avg Size | Status |
|---------|-------|----------|--------|
| 7.58 | 6,088 | 5,237B | ✅ Active (default) |
| latest | 6,089 | 5,059B | ✅ Active (boost: 0.90) |
| 7.57 | 5,808 | 5,026B | ✅ Active (boost: 0.95) |
| 7.56 | 5,605 | 4,498B | ✅ Active (boost: 0.90) |
| 7.55 | 5,154 | 4,146B | ✅ Active (boost: 0.85) |
| 7.54 | 4,905 | 4,052B | ✅ Active (boost: 0.80) |
| 7.53 | 4,680 | 3,992B | ✅ Active (boost: 0.75) |
| 7.52 | 4,572 | 3,931B | ✅ Active (boost: 0.70) |
| **Total** | **42,901** | **4,493B** | **8 versions** |
---
## 🚀 **Features**
### **✅ Standard Integration**
- **No special tools** - uses existing `search` like UI5, CAP, wdi5
- **63,454 total documents** indexed (up from 20,553)
- **30.52 MB FTS5 database** for lightning-fast search
### **✅ Intelligent Version Auto-Detection**
#### **Query Examples:**
```bash
# Version auto-detection from queries
"LOOP 7.57" → Searches ABAP 7.57 specifically
"SELECT latest" → Searches latest ABAP version
"exception handling 7.53" → Searches ABAP 7.53 specifically
"inline declarations" → Searches ABAP 7.58 (default)
"class definition 7.56" → Searches ABAP 7.56 specifically
```
#### **Results Show Correct Versions:**
```
Query: "LOOP 7.57"
✅ /abap-docs-757/abapcheck_loop (Score: 15.60)
✅ /abap-docs-757/abapexit_loop (Score: 15.60)
✅ /abap-docs-757/abenabap_loops (Score: 15.60)
Query: "SELECT latest"
✅ /abap-docs-latest/abenfree_selections (Score: 12.19)
✅ /abap-docs-latest/abenldb_selections (Score: 12.19)
✅ /abap-docs-latest/abapat_line-selection (Score: 12.10)
```
### **✅ Cross-Source Intelligence**
Finds related content across all SAP sources:
```
Query: "exception handling 7.53"
✅ ABAP 7.53 official docs (/abap-docs-753/)
✅ Clean ABAP style guides (/sap-styleguides/)
✅ ABAP cheat sheets (/abap-cheat-sheets/)
```
### **✅ Perfect LLM Experience**
- **Individual files** (1-10KB each) - perfect for context windows
- **Official attribution** - every file links to help.sap.com
- **Clean structure** - optimized markdown for LLM consumption
---
## 🔧 **Technical Implementation**
### **Metadata Configuration (27 Total Sources)**
```json
{
"sources": [
{ "id": "abap-docs-758", "boost": 0.95, "tags": ["abap", "7.58"] },
{ "id": "abap-docs-latest", "boost": 0.90, "tags": ["abap", "latest"] },
{ "id": "abap-docs-757", "boost": 0.95, "tags": ["abap", "7.57"] },
{ "id": "abap-docs-756", "boost": 0.90, "tags": ["abap", "7.56"] },
{ "id": "abap-docs-755", "boost": 0.85, "tags": ["abap", "7.55"] },
{ "id": "abap-docs-754", "boost": 0.80, "tags": ["abap", "7.54"] },
{ "id": "abap-docs-753", "boost": 0.75, "tags": ["abap", "7.53"] },
{ "id": "abap-docs-752", "boost": 0.70, "tags": ["abap", "7.52"] }
]
}
```
### **Context Boosting Strategy**
```typescript
"ABAP": {
"/abap-docs-758": 1.0, // Highest priority for general ABAP
"/abap-docs-latest": 0.98, // Latest features
"/abap-docs-757": 0.95, // Recent stable
"/abap-docs-756": 0.90, // Stable
// ... decreasing boost for older versions
}
```
### **URL Generation per Version**
```typescript
// Automatic version-specific URLs
"/abap-docs-757/abenloop.md"
→ "https://help.sap.com/doc/abapdocu_757_index_htm/7.57/en-US/abenloop.htm"
"/abap-docs-latest/abenselect.md"
→ "https://help.sap.com/doc/abapdocu_latest_index_htm/latest/en-US/abenselect.htm"
```
---
## 🎯 **Usage Patterns**
### **Version-Specific Queries**
```bash
# Search specific ABAP versions
search: "LOOP AT 7.57" # → ABAP 7.57 docs
search: "CDS views latest" # → Latest ABAP docs
search: "class definition 7.53" # → ABAP 7.53 docs
```
### **General ABAP Queries (Default 7.58)**
```bash
search: "SELECT statements" # → ABAP 7.58 docs
search: "internal tables" # → ABAP 7.58 docs
search: "exception handling" # → ABAP 7.58 docs
```
### **Cross-Source Results**
```bash
search: "inline declarations"
# Returns:
✅ Official ABAP docs (version-specific)
✅ Clean ABAP style guides
✅ ABAP cheat sheets
✅ Related UI5/CAP content
```
---
## 📈 **Performance & Quality**
### **Search Performance**
- **~50ms search time** (standard FTS5 performance)
- **63,454 total documents** in searchable index
- **30.52 MB database** - efficient storage
### **Result Quality**
- **Version-aware scoring** - newer versions get slight boost
- **Cross-source intelligence** - finds related content across all sources
- **LLM-optimized** - individual files perfect for context windows
### **Content Quality**
- **100% working links** - all JavaScript links fixed to help.sap.com URLs
- **Official attribution** - every file includes source documentation link
- **Clean structure** - optimized for LLM consumption
---
## 🔮 **Benefits of Standard Integration**
### **✅ Unified Experience**
- **One search tool** for all SAP development (ABAP + UI5 + CAP + testing)
- **Automatic version detection** - no need to specify versions manually
- **Cross-source results** - finds related content across documentation types
### **✅ Technical Excellence**
- **Standard architecture** - same proven system as UI5/CAP sources
- **No special tools** - uses existing infrastructure
- **Easy maintenance** - standard build and deployment process
### **✅ Developer Productivity**
- **42,901 individual ABAP files** ready for LLM consumption
- **8 versions supported** with intelligent prioritization
- **Perfect file sizes** (1-10KB) for optimal AI interaction
---
## 🎉 **Mission Complete: World's Most Comprehensive SAP MCP**
The SAP Docs MCP now provides:
- ✅ **Complete ABAP coverage** - 8 versions, 42,901+ files
- ✅ **Intelligent version detection** - auto-detects from queries
- ✅ **Unified interface** - one tool for all SAP development
- ✅ **Cross-source intelligence** - finds related content everywhere
- ✅ **LLM-optimized** - perfect file sizes and structure
- ✅ **Production-ready** - standard architecture, full testing
**The most advanced SAP development documentation system available for LLMs!** 🚀
```
--------------------------------------------------------------------------------
/src/lib/searchDb.ts:
--------------------------------------------------------------------------------
```typescript
import Database from "better-sqlite3";
import path from "path";
import { existsSync, statSync } from "fs";
import { CONFIG } from "./config.js";
let db: Database.Database | null = null;
export function openDb(dbPath?: string): Database.Database {
if (!db) {
// Use centralized config path
const defaultPath = path.join(process.cwd(), CONFIG.DB_PATH);
const finalPath = dbPath || defaultPath;
if (!existsSync(finalPath)) {
throw new Error(`FTS database not found at ${finalPath}. Run 'npm run build:fts' to create it.`);
}
db = new Database(finalPath, { readonly: true, fileMustExist: true });
// Read-only safe pragmas
db.pragma("query_only = ON");
db.pragma("cache_size = -8000"); // ~8MB page cache
}
return db;
}
export function closeDb(): void {
if (db) {
db.close();
db = null;
}
}
type Filters = {
libraries?: string[]; // e.g. ["/cap", "/sapui5"]
types?: string[]; // e.g. ["markdown","jsdoc","sample"]
};
export type FTSResult = {
id: string;
libraryId: string;
type: string;
title: string;
description: string;
relFile: string;
snippetCount: number;
bm25Score: number;
highlight: string;
};
export function toMatchQuery(userQuery: string): string {
// Convert user input into FTS syntax with prefix matching:
// keep quoted phrases as-is, append * to bare terms for prefix matching
const terms = userQuery.match(/"[^"]+"|\S+/g) ?? [];
// Very common stopwords that hurt FTS when ANDed together
const stopwords = new Set([
"a","an","the","to","in","on","for","and","or","of","with","from",
"how","what","why","when","where","which","who","whom","does","do","is","are"
]);
const cleanTerms = terms.map(t => {
if (t.startsWith('"') && t.endsWith('"')) return t; // phrase query
// For terms with dots (like sap.m.Button), quote them as phrases
if (t.includes('.')) {
return `"${t}"`;
}
// Handle annotation qualifiers with # (like #SpecificationWidthColumnChart)
if (t.startsWith('#') && t.length > 1) {
// Keep the # and treat as exact phrase for better matching
return `"${t}"`;
}
// Handle compound terms with # (like UI.Chart#Something)
if (t.includes('#') && !t.startsWith('#')) {
// Split on # and treat as phrase to preserve structure
return `"${t}"`;
}
// Sanitize and add prefix matching for simple terms
const clean = t.replace(/[^\w]/g, "").toLowerCase();
if (!clean || stopwords.has(clean)) return "";
return `${clean}*`;
}).filter(Boolean);
// Use OR logic for better recall in BM25-only mode (configurable)
// FTS5 will still rank documents with more matching terms higher
if (CONFIG.USE_OR_LOGIC || cleanTerms.length > 3) {
return cleanTerms.join(" OR ");
}
return cleanTerms.join(" ");
}
/**
* Fast FTS5 candidate filtering
* Returns document IDs that match the query, for use with existing sophisticated scoring
*/
export function getFTSCandidateIds(userQuery: string, filters: Filters = {}, limit = 100): string[] {
const database = openDb();
const match = toMatchQuery(userQuery);
if (!match.trim()) {
return []; // Empty query
}
// Build WHERE conditions
const conditions = ["docs MATCH ?"];
const params: any[] = [match];
if (filters.libraries?.length) {
const placeholders = filters.libraries.map(() => "?").join(",");
conditions.push(`libraryId IN (${placeholders})`);
params.push(...filters.libraries);
}
if (filters.types?.length) {
const placeholders = filters.types.map(() => "?").join(",");
conditions.push(`type IN (${placeholders})`);
params.push(...filters.types);
}
const sql = `
SELECT id
FROM docs
WHERE ${conditions.join(" AND ")}
ORDER BY bm25(docs)
LIMIT ?
`;
try {
const stmt = database.prepare(sql);
const rows = stmt.all(...params, limit) as { id: string }[];
return rows.map(r => r.id);
} catch (error) {
console.warn("FTS query failed, falling back to full search:", error);
return []; // Fallback gracefully
}
}
/**
* Full FTS search with results (for debugging/testing)
*/
export function searchFTS(userQuery: string, filters: Filters = {}, limit = 20): FTSResult[] {
const database = openDb();
const match = toMatchQuery(userQuery);
if (!match.trim()) {
return [];
}
// Build WHERE conditions
const conditions = ["docs MATCH ?"];
const params: any[] = [match];
if (filters.libraries?.length) {
const placeholders = filters.libraries.map(() => "?").join(",");
conditions.push(`libraryId IN (${placeholders})`);
params.push(...filters.libraries);
}
if (filters.types?.length) {
const placeholders = filters.types.map(() => "?").join(",");
conditions.push(`type IN (${placeholders})`);
params.push(...filters.types);
}
// BM25 weights: title, description, keywords, controlName, namespace
// Higher weight = more important (title and controlName are most important)
const sql = `
SELECT
id, libraryId, type, title, description, relFile, snippetCount,
highlight(docs, 2, '<mark>', '</mark>') AS highlight,
bm25(docs, 1.0, 8.0, 2.0, 4.0, 6.0, 3.0) AS bm25Score
FROM docs
WHERE ${conditions.join(" AND ")}
ORDER BY bm25Score
LIMIT ?
`;
try {
const stmt = database.prepare(sql);
const rows = stmt.all(...params, limit) as any[];
return rows.map(r => ({
id: r.id,
libraryId: r.libraryId,
type: r.type,
title: r.title,
description: r.description,
relFile: r.relFile,
snippetCount: r.snippetCount,
bm25Score: Number(r.bm25Score),
highlight: r.highlight || r.title
}));
} catch (error) {
console.warn("FTS query failed:", error);
return [];
}
}
/**
* Get database stats for monitoring
*/
export function getFTSStats(): { rowCount: number; dbSize: number; mtime: string } | null {
try {
const database = openDb();
const rowCount = database.prepare("SELECT count(*) as n FROM docs").get() as { n: number };
const dbPath = path.join(process.cwd(), CONFIG.DB_PATH);
const stats = statSync(dbPath);
return {
rowCount: rowCount.n,
dbSize: stats.size,
mtime: stats.mtime.toISOString()
};
} catch (error) {
console.warn("Could not get FTS stats:", error);
return null;
}
}
```
--------------------------------------------------------------------------------
/docs/CONTENT-SIZE-LIMITS.md:
--------------------------------------------------------------------------------
```markdown
# Content Size Limits
## Overview
To ensure optimal performance and prevent token overflow in LLM interactions, the server implements intelligent content size limits for SAP Help Portal and Community content retrieval.
## Configuration
### Maximum Content Length
**Default: 75,000 characters** (~18,750 tokens)
This limit is configurable in `/src/lib/config.ts`:
```typescript
export const CONFIG = {
// Maximum content length for SAP Help and Community full content retrieval
// Limits help prevent token overflow and keep responses manageable (~18,750 tokens)
MAX_CONTENT_LENGTH: 75000, // 75,000 characters
};
```
## Affected Tools
The content size limit applies to the following MCP tools:
### 1. `sap_help_get`
Retrieves full SAP Help Portal pages. If content exceeds 75,000 characters, it is intelligently truncated while preserving:
- Beginning section (introduction and main content)
- End section (conclusions and examples)
- A clear truncation notice showing what was omitted
### 2. `sap_community_search`
Returns full content of top 3 SAP Community posts. Each post is truncated if needed using the same intelligent algorithm.
### 3. Community Post Retrieval
Individual community posts fetched via `fetch` tool with `community-*` IDs are also subject to truncation.
## Intelligent Truncation Algorithm
When content exceeds the maximum length, the truncation algorithm:
### Preservation Strategy
- **60%** from the beginning (introduction, overview, main content)
- **20%** from the end (conclusions, examples, summaries)
- **20%** reserved for truncation notice and natural break padding
### Natural Boundaries
The algorithm attempts to break content at natural points rather than mid-sentence:
1. Paragraph breaks (`\n\n`)
2. Markdown headings (`# Heading`)
3. Code block boundaries (` ```\n`)
4. Horizontal rules (`---`)
5. Sentence endings (`. `)
### Truncation Notice
A clear notice is inserted showing:
- Original content length in characters
- Approximate original token count (chars ÷ 4)
- Number of omitted characters
- Percentage of content omitted
- Explanation that beginning and end are preserved
Example truncation notice:
```markdown
---
⚠️ **Content Truncated**
The full content was 425,000 characters (approximately 106,250 tokens).
For readability and performance, 350,000 characters (82%) have been omitted from the middle section.
The beginning and end of the document are preserved above and below this notice.
---
```
## Rationale
### Why 75,000 Characters?
1. **LLM Context Windows**: Fits comfortably in most modern LLM context windows:
- Claude 3.5 Sonnet: 200k tokens (can handle ~800k chars)
- GPT-4 Turbo: 128k tokens (can handle ~512k chars)
- Leaves room for conversation history and system prompts
2. **Performance**: Reduces response time and API costs while maintaining comprehensive coverage
3. **Readability**: Very long documents (>100k chars) are often better consumed in multiple focused queries
4. **Practical Coverage**: 75k characters is sufficient for most documentation pages while preventing extreme cases
### Alternative Approaches Considered
| Approach | Characters | Tokens (approx) | Trade-off |
|----------|-----------|-----------------|-----------|
| Conservative | 50,000 | ~12,500 | Too restrictive for comprehensive docs |
| **Current** | **75,000** | **~18,750** | **Balanced - recommended** |
| Generous | 100,000 | ~25,000 | Risk of slow responses |
| Maximum | 150,000 | ~37,500 | Only for edge cases |
## Implementation Details
### Source Files
- **Configuration**: `/src/lib/config.ts` - MAX_CONTENT_LENGTH constant
- **Truncation Logic**: `/src/lib/truncate.ts` - Intelligent truncation implementation
- **SAP Help**: `/src/lib/sapHelp.ts` - Applied in `getSapHelpContent()`
- **Community**: `/src/lib/communityBestMatch.ts` - Applied in post retrieval functions
### Functions
#### `truncateContent(content: string, maxLength?: number): TruncationResult`
Main truncation function with beginning/end preservation.
**Returns:**
```typescript
{
content: string; // Truncated content
wasTruncated: boolean; // Whether truncation occurred
originalLength: number; // Original character count
truncatedLength: number; // Final character count
}
```
#### `truncateContentSimple(content: string, maxLength?: number): TruncationResult`
Alternative truncation function that only preserves beginning with end notice.
## Monitoring and Adjustment
### When to Increase Limit
Consider increasing if:
- Users frequently encounter truncated content
- Average document sizes are near the limit
- LLM context windows have increased significantly
### When to Decrease Limit
Consider decreasing if:
- Response times are too slow
- Token costs are concerning
- Most content doesn't use the available space
### Override for Specific Cases
To override the limit for specific use cases, modify the `truncateContent()` call:
```typescript
// Custom limit of 100,000 characters
const truncationResult = truncateContent(fullContent, 100000);
```
## User Experience
### Transparent Communication
When content is truncated, users see:
- Clear visual indicator (⚠️ warning emoji)
- Exact statistics (original length, omitted amount, percentage)
- Explanation of what's preserved
- No disruption to markdown formatting
### Best Practices for Users
1. **Specific Queries**: Ask focused questions to get relevant sections
2. **Multiple Requests**: Break very long documents into targeted fetches
3. **Search First**: Use `sap_help_search` to find specific sections before fetching
4. **Check URLs**: Visit the provided URLs for complete untruncated content
## Future Enhancements
Potential improvements to consider:
1. **Dynamic Limits**: Adjust based on LLM context window
2. **Sectioned Retrieval**: Fetch specific document sections
3. **Summary Generation**: Auto-summarize omitted middle sections
4. **User Preferences**: Allow users to specify their preferred limits
5. **Compression**: Apply content compression for technical reference material
## Testing
Content size limits are tested in:
- Unit tests for truncation functions
- Integration tests for SAP Help and Community tools
- Manual validation with known large documents
## Related Documentation
- **Architecture**: `/docs/ARCHITECTURE.md` - System overview
- **Tool Descriptions**: `/docs/CURSOR-SETUP.md` - MCP tool documentation
- **Community Search**: `/docs/COMMUNITY-SEARCH-IMPLEMENTATION.md` - Community integration details
```
--------------------------------------------------------------------------------
/src/lib/url-generation/utils.ts:
--------------------------------------------------------------------------------
```typescript
/**
* Common utilities for URL generation across different documentation sources
*/
export interface FrontmatterData {
id?: string;
slug?: string;
title?: string;
sidebar_label?: string;
[key: string]: any;
}
/**
* Extract frontmatter from document content
* Supports YAML frontmatter format used in Markdown/MDX files
*/
export function parseFrontmatter(content: string): FrontmatterData {
const frontmatterMatch = content.match(/^---\s*\n([\s\S]*?)\n---/);
if (!frontmatterMatch) {
return {};
}
const frontmatter = frontmatterMatch[1];
const result: FrontmatterData = {};
// Parse simple key-value pairs
const lines = frontmatter.split('\n');
let currentKey = '';
let isInArray = false;
for (const line of lines) {
const trimmedLine = line.trim();
if (!trimmedLine || trimmedLine.startsWith('#')) {
continue; // Skip empty lines and comments
}
// Handle array items (lines starting with -)
if (trimmedLine.startsWith('-')) {
if (isInArray && currentKey) {
const arrayValue = trimmedLine.substring(1).trim();
if (!Array.isArray(result[currentKey])) {
result[currentKey] = [];
}
(result[currentKey] as string[]).push(arrayValue);
}
continue;
}
// Handle key-value pairs
const colonIndex = trimmedLine.indexOf(':');
if (colonIndex !== -1) {
currentKey = trimmedLine.substring(0, colonIndex).trim();
const value = trimmedLine.substring(colonIndex + 1).trim();
if (value === '') {
// This might be the start of an array
isInArray = true;
result[currentKey] = [];
} else {
isInArray = false;
// Clean up quoted values
result[currentKey] = value.replace(/^["']|["']$/g, '');
}
}
}
return result;
}
/**
* Detect the main section/topic from content for anchor generation
*/
export function detectContentSection(content: string, anchorStyle: 'docsify' | 'github' | 'custom'): string | null {
// Find the first major heading (## or #) that gives context about the content
const headingMatch = content.match(/^#{1,2}\s+(.+)$/m);
if (!headingMatch) {
return null;
}
const heading = headingMatch[1].trim();
// Convert heading to anchor format based on style
switch (anchorStyle) {
case 'docsify':
// Docsify format: lowercase, spaces to hyphens, remove special chars
return heading
.toLowerCase()
.replace(/[^\w\s-]/g, '') // Remove special characters except hyphens
.replace(/\s+/g, '-') // Spaces to hyphens
.replace(/-+/g, '-') // Multiple hyphens to single
.replace(/^-|-$/g, ''); // Remove leading/trailing hyphens
case 'github':
// GitHub format: lowercase, spaces to hyphens, keep some special chars
return heading
.toLowerCase()
.replace(/[^\w\s-]/g, '')
.replace(/\s+/g, '-');
case 'custom':
default:
// Return as-is for custom handling
return heading;
}
}
/**
* Determine the section path from file relative path
*/
export function extractSectionFromPath(relFile: string): string {
if (relFile.includes('guides/')) {
return '/guides/';
} else if (relFile.includes('features/')) {
return '/features/';
} else if (relFile.includes('tutorials/')) {
return '/tutorials/';
} else if (relFile.includes('environments/')) {
return '/environments/';
} else if (relFile.includes('getting-started/')) {
return '/getting-started/';
} else if (relFile.includes('examples/')) {
return '/examples/';
} else if (relFile.includes('api/')) {
return '/api/';
}
return '';
}
/**
* Clean filename for URL usage
*/
export function cleanFilename(filename: string): string {
return filename
.replace(/\.mdx?$/, '') // Remove .md/.mdx extensions
.replace(/\.html?$/, '') // Remove .html/.htm extensions
.replace(/\s+/g, '-') // Spaces to hyphens
.toLowerCase();
}
/**
* Build URL with proper path joining
*/
export function buildUrl(baseUrl: string, ...pathSegments: string[]): string {
const cleanBase = baseUrl.replace(/\/$/, ''); // Remove trailing slash
const cleanSegments = pathSegments
.filter(segment => segment && segment.trim() !== '') // Remove empty segments
.map(segment => segment.replace(/^\/|\/$/g, '')); // Remove leading/trailing slashes
if (cleanSegments.length === 0) {
return cleanBase;
}
return `${cleanBase}/${cleanSegments.join('/')}`;
}
/**
* Extract library ID from document ID path
* Used for search result URL generation
*/
export function extractLibraryIdFromPath(docId: string): string {
if (docId.startsWith('/')) {
const parts = docId.split('/');
return parts.length > 1 ? `/${parts[1]}` : docId;
}
return docId;
}
/**
* Extract relative file path from document ID
* Used for search result URL generation
*/
export function extractRelativeFileFromPath(docId: string): string {
if (docId.includes('/')) {
const parts = docId.split('/');
return parts.length > 2 ? parts.slice(2).join('/') : '';
}
return '';
}
/**
* Format a single search result with URL generation and excerpt truncation
* Shared utility for consistent search result formatting across servers
*/
export function formatSearchResult(
result: any,
excerptLength: number,
urlGenerator?: {
generateDocumentationUrl: (libraryId: string, relFile: string, content: string, config: any) => string | null;
getDocUrlConfig: (libraryId: string) => any;
}
): string {
// Extract library ID and relative file path to generate URL
const libraryId = result.sourceId ? `/${result.sourceId}` : extractLibraryIdFromPath(result.id);
const relFile = extractRelativeFileFromPath(result.id);
// Try to generate documentation URL
let urlInfo = '';
if (urlGenerator) {
try {
const config = urlGenerator.getDocUrlConfig && urlGenerator.getDocUrlConfig(libraryId);
if (config && urlGenerator.generateDocumentationUrl) {
const docUrl = urlGenerator.generateDocumentationUrl(libraryId, relFile, result.text || '', config);
if (docUrl) {
urlInfo = `\n 🔗 ${docUrl}`;
}
}
} catch (error) {
// Silently fail URL generation
console.warn(`URL generation failed for ${libraryId}/${relFile}:`, error);
}
}
return `⭐️ **${result.id}** (Score: ${result.finalScore.toFixed(2)})\n ${(result.text || '').substring(0, excerptLength)}${urlInfo}\n Use in fetch\n`;
}
```
--------------------------------------------------------------------------------
/docs/COMMUNITY-SEARCH-IMPLEMENTATION.md:
--------------------------------------------------------------------------------
```markdown
# SAP Community Search Implementation
## Overview
The SAP Community search has been completely rewritten to use HTML scraping instead of the LiQL API approach, providing better search results that match the SAP Community's "Best Match" ranking algorithm.
## New Implementation Details
### 1. HTML Scraping Module (`src/lib/communityBestMatch.ts`)
**Key Features:**
- Direct HTML scraping of SAP Community search results
- Extracts comprehensive metadata: title, author, publish date, likes, snippet, tags
- Zero external dependencies - uses native Node.js `fetch` and regex parsing
- Respects SAP Community's "Best Match" ranking
- Includes both search and full post retrieval functions
**Functions:**
- `searchCommunityBestMatch(query, options)` - Search for community posts via HTML scraping
- `getCommunityPostByUrl(url, userAgent)` - Get full post content from URL (fallback method)
- `getCommunityPostsByIds(postIds, userAgent)` - **NEW**: Batch retrieve multiple posts via LiQL API
- `getCommunityPostById(postId, userAgent)` - **NEW**: Single post retrieval via LiQL API
- `searchAndGetTopPosts(query, topN, options)` - **NEW**: Search + batch retrieve in one call
### 2. Updated Search Integration (`src/lib/localDocs.ts`)
**Changes:**
- Replaced LiQL API calls with HTML scraping
- Enhanced SearchResult interface with new fields (author, likes, tags)
- Improved post ID handling for both legacy and new URL-based formats
- Better error handling and graceful fallbacks
### 3. Enhanced Type Definitions (`src/lib/types.ts`)
**New SearchResult fields:**
- `author?: string` - Post author name
- `likes?: number` - Number of kudos/likes
- `tags?: string[]` - Associated topic tags
## Usage
### Search Community Posts
```javascript
import { searchCommunityBestMatch } from './src/lib/communityBestMatch.js';
const results = await searchCommunityBestMatch('SAPUI5 wizard', {
includeBlogs: true,
limit: 10,
userAgent: 'MyApp/1.0'
});
```
### Batch Retrieve Multiple Posts (Recommended)
```javascript
import { getCommunityPostsByIds } from './src/lib/communityBestMatch.js';
// Efficient batch retrieval using LiQL API
const posts = await getCommunityPostsByIds(['13961398', '13446100', '14152848'], 'MyApp/1.0');
// Returns: { '13961398': 'formatted content...', '13446100': 'formatted content...', ... }
```
### Search + Get Top Posts (One-Stop Solution)
```javascript
import { searchAndGetTopPosts } from './src/lib/communityBestMatch.js';
// Search and get full content of top 3 posts in one call
const { search, posts } = await searchAndGetTopPosts('odata cache', 3, {
includeBlogs: true,
userAgent: 'MyApp/1.0'
});
search.forEach((result, index) => {
console.log(`${index + 1}. ${result.title}`);
if (posts[result.postId]) {
console.log(posts[result.postId]); // Full formatted content
}
});
```
### Single Post Retrieval
```javascript
import { getCommunityPostById } from './src/lib/communityBestMatch.js';
const content = await getCommunityPostById('13961398', 'MyApp/1.0');
```
### Fallback: Get Full Post Content by URL
```javascript
import { getCommunityPostByUrl } from './src/lib/communityBestMatch.js';
const content = await getCommunityPostByUrl(
'https://community.sap.com/t5/technology-blogs-by-sap/...',
'MyApp/1.0'
);
```
### Via MCP Server
The community search is exposed as the `sap_community_search` tool, which now **automatically returns the full content** of the top 3 most relevant posts using the efficient LiQL API batch retrieval. Individual posts can also be retrieved using `fetch` with community post IDs.
**Key Behavior:**
- **`sap_community_search`**: Returns full content of top 3 posts (search + batch retrieval in one call)
- **`fetch`**: Retrieves individual post content by ID
## Testing
### Run the Test Suite
```bash
# Run comprehensive test suite (recommended)
npm run test:community
# Run directly with Node.js (TypeScript support)
node test/community-search.ts
```
The **unified test suite** (`test/community-search.ts`) covers:
- **HTML Search Scraping**: Search accuracy, post ID extraction, metadata parsing
- **LiQL API Batch Retrieval**: Efficient multi-post content retrieval
- **Single Post Retrieval**: Individual post fetching via API
- **Convenience Functions**: Combined search + batch retrieval workflow
- **Direct API Testing**: Raw LiQL API validation
- **Known Post Validation**: Testing with specific real posts
### Test Features
- **TypeScript**: Modern, type-safe test implementation
- **Comprehensive Coverage**: All functionality tested in one script
- **Organized Structure**: Modular test functions with clear separation
- **Real-time Validation**: Tests against live SAP Community data
- **Error Handling**: Robust error reporting and graceful failures
## Benefits of the New Implementation
1. **Better Search Results**: Uses SAP Community's native "Best Match" algorithm
2. **Richer Metadata**: Extracts author, likes, tags, and better snippets
3. **Efficient Batch Retrieval**: LiQL API for fast bulk post content retrieval
4. **Hybrid Approach**: HTML scraping for search + API calls for content = best of both worlds
5. **One-Stop Functions**: `searchAndGetTopPosts()` combines search + retrieval in single call
6. **Improved Reliability**: Fallback methods for different scenarios
7. **Real-time Data**: Gets the same results users see on the website
## Technical Notes
### HTML Parsing Strategy
- Uses regex patterns to extract structured data from Khoros-based SAP Community
- Targets stable CSS classes and HTML structure patterns
- Includes fallback patterns for different page layouts
- Sanitizes and decodes HTML entities properly
### Rate Limiting & Respect
- Includes User-Agent identification
- Test script includes delays between requests
- Graceful error handling for HTTP failures
- Respects community guidelines
### Post ID Formats
The system now supports two post ID formats:
1. **Legacy**: `community-postId` (tries to construct URL)
2. **New**: `community-url-encodedUrl` (direct URL extraction)
### Error Handling
- Network failures return empty results instead of crashing
- HTML parsing errors are logged but don't break the search
- Malformed URLs are handled gracefully
- User-Agent can be customized for identification
## Future Enhancements
1. **Caching**: Add optional caching layer for frequently accessed posts
2. **Pagination**: Support for multiple result pages
3. **Advanced Filtering**: Filter by author, date range, or specific tags
4. **Performance**: Add connection pooling for high-volume usage
## Migration Notes
The new implementation is a **drop-in replacement** for the old LiQL-based approach:
- Same function signatures for `searchCommunity()`
- Same MCP tool interface (`sap_community_search`)
- Enhanced with additional metadata fields
- Backward compatible post ID handling
```
--------------------------------------------------------------------------------
/src/lib/logger.ts:
--------------------------------------------------------------------------------
```typescript
// src/lib/logger.ts
// Standard logging utility with configurable levels
export enum LogLevel {
ERROR = 0,
WARN = 1,
INFO = 2,
DEBUG = 3
}
export class Logger {
private level: LogLevel;
private enableJson: boolean;
private startTime: number = Date.now();
constructor() {
// Standard environment-based configuration
const envLevel = process.env.LOG_LEVEL?.toUpperCase() || 'INFO';
this.level = LogLevel[envLevel as keyof typeof LogLevel] ?? LogLevel.INFO;
this.enableJson = process.env.LOG_FORMAT === 'json';
// Setup global error handlers
this.setupGlobalErrorHandlers();
}
private shouldLog(level: LogLevel): boolean {
return level <= this.level;
}
private formatMessage(level: string, message: string, meta?: Record<string, any>): string {
const timestamp = new Date().toISOString();
if (this.enableJson) {
return JSON.stringify({
timestamp,
level,
message,
...meta
});
} else {
const metaStr = meta ? ` ${JSON.stringify(meta)}` : '';
return `${timestamp} [${level}] ${message}${metaStr}`;
}
}
error(message: string, meta?: Record<string, any>): void {
if (this.shouldLog(LogLevel.ERROR)) {
// Always use stderr for MCP stdio compatibility
process.stderr.write(this.formatMessage('ERROR', message, meta) + '\n');
}
}
warn(message: string, meta?: Record<string, any>): void {
if (this.shouldLog(LogLevel.WARN)) {
// Always use stderr for MCP stdio compatibility
process.stderr.write(this.formatMessage('WARN', message, meta) + '\n');
}
}
info(message: string, meta?: Record<string, any>): void {
if (this.shouldLog(LogLevel.INFO)) {
// Always use stderr for MCP stdio compatibility
process.stderr.write(this.formatMessage('INFO', message, meta) + '\n');
}
}
debug(message: string, meta?: Record<string, any>): void {
if (this.shouldLog(LogLevel.DEBUG)) {
// Always use stderr for MCP stdio compatibility
process.stderr.write(this.formatMessage('DEBUG', message, meta) + '\n');
}
}
private sanitizeQuery(query: string): string {
// Basic sanitization for logging
return query
.replace(/\b\d{4,}\b/g, '[NUM]')
.replace(/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g, '[EMAIL]')
.substring(0, 200);
}
private sanitizeError(error: string): string {
return error
.replace(/\/[^\s]+/g, '[PATH]')
.replace(/\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/g, '[IP]')
.substring(0, 300);
}
// Setup global error handlers to catch unhandled errors
private setupGlobalErrorHandlers(): void {
// Handle unhandled promise rejections
process.on('unhandledRejection', (reason: any, promise: Promise<any>) => {
this.error('Unhandled Promise Rejection', {
reason: this.sanitizeError(String(reason)),
stack: reason?.stack ? this.sanitizeError(reason.stack) : undefined,
pid: process.pid,
uptime: Date.now() - this.startTime,
timestamp: new Date().toISOString()
});
});
// Handle uncaught exceptions
process.on('uncaughtException', (error: Error) => {
this.error('Uncaught Exception', {
message: this.sanitizeError(error.message),
stack: this.sanitizeError(error.stack || ''),
name: error.name,
pid: process.pid,
uptime: Date.now() - this.startTime,
timestamp: new Date().toISOString()
});
// Exit after logging the error
setTimeout(() => process.exit(1), 100);
});
// Handle warnings (useful for debugging deprecations and other issues)
process.on('warning', (warning: Error) => {
this.warn('Process Warning', {
message: warning.message,
name: warning.name,
stack: warning.stack ? this.sanitizeError(warning.stack) : undefined,
pid: process.pid,
timestamp: new Date().toISOString()
});
});
}
// Enhanced tool execution logging with timing
logToolStart(tool: string, query: string, clientInfo?: Record<string, any>): { startTime: number; requestId: string } {
const startTime = Date.now();
const requestId = `req_${startTime}_${Math.random().toString(36).substr(2, 9)}`;
this.info('Tool execution started', {
tool,
query: this.sanitizeQuery(query),
client: clientInfo,
requestId,
timestamp: new Date().toISOString(),
pid: process.pid,
uptime: Date.now() - this.startTime
});
return { startTime, requestId };
}
logToolSuccess(tool: string, requestId: string, startTime: number, resultCount?: number, additionalInfo?: Record<string, any>): void {
const duration = Date.now() - startTime;
this.info('Tool execution completed', {
tool,
requestId,
duration,
resultCount,
...additionalInfo,
timestamp: new Date().toISOString(),
pid: process.pid
});
}
logToolError(tool: string, requestId: string, startTime: number, error: any, fallback?: boolean): void {
const duration = Date.now() - startTime;
this.error('Tool execution failed', {
tool,
requestId,
duration,
error: this.sanitizeError(String(error)),
stack: error?.stack ? this.sanitizeError(error.stack) : undefined,
errorName: error?.name,
fallback: fallback || false,
timestamp: new Date().toISOString(),
pid: process.pid,
uptime: Date.now() - this.startTime
});
}
// Enhanced request logging with more context
logRequest(tool: string, query: string, clientInfo?: Record<string, any>): void {
this.info('Tool request received', {
tool,
query: this.sanitizeQuery(query),
client: {
...clientInfo,
userAgent: clientInfo?.headers?.['user-agent'],
contentType: clientInfo?.headers?.['content-type']
},
timestamp: new Date().toISOString(),
pid: process.pid,
uptime: Date.now() - this.startTime
});
}
// Log transport/connection issues
logTransportEvent(event: string, sessionId?: string, details?: Record<string, any>): void {
this.info('Transport event', {
event,
sessionId,
details,
timestamp: new Date().toISOString(),
pid: process.pid,
uptime: Date.now() - this.startTime
});
}
// Log memory and performance metrics
logPerformanceMetrics(): void {
const memUsage = process.memoryUsage();
const cpuUsage = process.cpuUsage();
this.debug('Performance metrics', {
memory: {
rss: Math.round(memUsage.rss / 1024 / 1024),
heapTotal: Math.round(memUsage.heapTotal / 1024 / 1024),
heapUsed: Math.round(memUsage.heapUsed / 1024 / 1024),
external: Math.round(memUsage.external / 1024 / 1024)
},
cpu: {
user: cpuUsage.user,
system: cpuUsage.system
},
uptime: Math.round((Date.now() - this.startTime) / 1000),
pid: process.pid,
timestamp: new Date().toISOString()
});
}
}
// Export singleton logger instance
export const logger = new Logger();
```
--------------------------------------------------------------------------------
/docs/DEV.md:
--------------------------------------------------------------------------------
```markdown
# 🛠️ Development Guide
## Quick Start
### 🚀 **Initial Setup**
```bash
# Clone and install
git clone <repo-url>
cd sap-docs-mcp
npm install
# Run enhanced setup (submodules + build)
npm run setup
# Start development server
npm run start:http
```
### 🧪 **Run Tests**
```bash
npm run test:smoke # Quick validation
npm run test:fast # Skip build, test only
npm run test # Full build + test
```
## Common Commands
### 📦 **Build Commands**
```bash
npm run build:tsc # Compile TypeScript
npm run build:index # Build documentation index
npm run build:fts # Build FTS5 search database
npm run build # Complete build pipeline (tsc + index + fts)
```
### 🖥️ **Server Commands**
```bash
npm start # MCP stdio server (for Claude)
npm run start:http # HTTP development server (port 3001)
npm run start:streamable # Streamable HTTP server (port 3122)
```
### 🧪 **Test Commands**
```bash
npm run test:smoke # Quick smoke tests
npm run test:fast # Test without rebuild
npm run test # Full test suite
npm run test:community # SAP Community search tests
npm run inspect # MCP protocol inspector
```
## Environment Variables
### 🔧 **Core Configuration**
```bash
RETURN_K=25 # Number of search results (default: 25)
LOG_LEVEL=INFO # Logging level (ERROR, WARN, INFO, DEBUG)
LOG_FORMAT=json # Log format (json or text)
NODE_ENV=production # Environment mode
```
### 🗄️ **Database & Paths**
```bash
DB_PATH=dist/data/docs.sqlite # FTS5 database path
METADATA_PATH=src/metadata.json # Metadata configuration path
```
### 🌐 **Server Configuration**
```bash
PORT=3001 # HTTP server port
MCP_PORT=3122 # Streamable HTTP MCP port
```
## Development Servers
### 📡 **1. Stdio MCP Server** (Main)
```bash
npm run start:stdio
# For Claude/LLM integration via stdio transport
```
### 🌐 **2. HTTP Development Server**
```bash
npm run start:http
# Access: http://localhost:3001
# Endpoints: /status, /healthz, /readyz, /mcp
```
### 🔄 **3. Streamable HTTP Server**
```bash
npm run start:streamable
# Access: http://localhost:3122
# Endpoints: /mcp, /health
```
## Where to Change Things
### 🔍 **Search Behavior**
- **Query Processing**: `src/lib/searchDb.ts` → `toMatchQuery()`
- **Search Logic**: `src/lib/search.ts` → `search()`
- **Result Formatting**: `src/lib/localDocs.ts` → `searchLibraries()`
### ⚙️ **Configuration**
- **Source Settings**: `src/metadata.json` → Add/modify sources
- **Core Config**: `src/lib/config.ts` → System settings
- **Metadata APIs**: `src/lib/metadata.ts` → Configuration access
### 🛠️ **MCP Tools**
- **Tool Definitions**: `src/server.ts` → `ListToolsRequestSchema`
- **Tool Handlers**: `src/server.ts` → `CallToolRequestSchema`
- **HTTP Endpoints**: `src/http-server.ts` → `/mcp` handler
### 🏗️ **Build Process**
- **Index Building**: `scripts/build-index.ts`
- **FTS Database**: `scripts/build-fts.ts`
- **Source Processing**: Modify build scripts for new source types
### 🧪 **Tests**
- **Test Cases**: `test/tools/search/` → Add new test files
- **Test Runner**: `test/tools/run-tests.js` → Modify test execution
- **Output Parsing**: `test/_utils/parseResults.js` → Update format expectations
### 🚀 **Deployment**
- **PM2 Config**: `ecosystem.config.cjs` → Process configuration
- **GitHub Actions**: `.github/workflows/deploy-mcp-sap-docs.yml`
- **Setup Script**: `setup.sh` → Deployment automation
## Adding New Documentation Sources
### 1. **Update Metadata** (`src/metadata.json`)
```json
{
"id": "new-source",
"type": "documentation",
"libraryId": "/new-source",
"sourcePath": "new-source/docs",
"baseUrl": "https://example.com/docs",
"pathPattern": "/{file}",
"anchorStyle": "github",
"boost": 0.05,
"tags": ["new", "documentation"],
"description": "New documentation source"
}
```
### 2. **Add Context Boosts** (if needed)
```json
"contextBoosts": {
"New Context": {
"/new-source": 1.0,
"/other-source": 0.3
}
}
```
### 3. **Add Library Mapping** (if needed)
```json
"libraryMappings": {
"new-source-alias": "new-source"
}
```
### 4. **No Code Changes Required!**
The metadata APIs automatically handle the new source.
## Debugging
### 🔍 **Search Issues**
```bash
# Test specific queries
node -e "
import { search } from './dist/src/lib/search.js';
const results = await search('your query');
console.log(JSON.stringify(results, null, 2));
"
# Check FTS database
sqlite3 dist/data/docs.sqlite "SELECT * FROM docs WHERE docs MATCH 'your query' LIMIT 5;"
```
### 📊 **Metadata Issues**
```bash
# Test metadata loading
node -e "
import { loadMetadata, getSourceBoosts } from './dist/src/lib/metadata.js';
loadMetadata();
console.log('Boosts:', getSourceBoosts());
"
```
### 🌐 **Server Issues**
```bash
# Check server health
curl http://localhost:3001/status
curl http://localhost:3122/health
# Test search endpoint
curl -X POST http://localhost:3001/mcp \
-H "Content-Type: application/json" \
-d '{"role": "user", "content": "wizard"}'
```
## Performance Optimization
### ⚡ **Search Performance**
- **FTS5 Tuning**: Modify `scripts/build-fts.ts` for different indexing strategies
- **Query Optimization**: Adjust `toMatchQuery()` in `src/lib/searchDb.ts`
- **Result Limits**: Configure `RETURN_K` environment variable
### 💾 **Memory Usage**
- **Index Size**: Monitor `dist/data/` artifact sizes
- **Metadata Loading**: Lazy loading in `src/lib/metadata.ts`
- **Process Monitoring**: Use PM2 monitoring features
## Common Issues
### ❌ **Build Failures**
```bash
# Clean and rebuild
rm -rf dist/
npm run build:all
```
### ❌ **Search Returns No Results**
```bash
# Check if database exists
ls -la dist/data/docs.sqlite
# Verify index content
sqlite3 dist/data/docs.sqlite "SELECT COUNT(*) FROM docs;"
```
### ❌ **Metadata Loading Errors**
```bash
# Validate JSON syntax
node -e "JSON.parse(require('fs').readFileSync('src/metadata.json', 'utf8'))"
# Check file permissions
ls -la src/metadata.json
```
### ❌ **Server Won't Start**
```bash
# Check port availability
lsof -i :3001
lsof -i :3122
# Kill conflicting processes
lsof -ti:3001 | xargs kill -9
```
## Best Practices
### 📝 **Code Changes**
1. **Update Cursor Rules**: Modify `.cursor/rules/` when changing architecture
2. **Test First**: Run smoke tests before committing
3. **Metadata Over Code**: Use metadata.json for configuration changes
4. **Type Safety**: Use metadata APIs, never direct JSON access
### 🧪 **Testing**
1. **Smoke Tests**: Always run before deployment
2. **Integration Tests**: Test full MCP tool workflows
3. **Performance Tests**: Monitor search response times
4. **Output Validation**: Ensure format consistency
### 🚀 **Deployment**
1. **Build Validation**: Ensure all artifacts generated
2. **Health Checks**: Verify all endpoints after deployment
3. **Rollback Plan**: Keep previous artifacts for quick rollback
4. **Monitoring**: Watch logs and performance metrics
## Useful Development Tools
### 🔧 **VS Code Extensions**
- **REST Client**: Use `test-search.http` for API testing
- **SQLite Viewer**: Inspect FTS5 database content
- **JSON Schema**: Validate metadata.json structure
### 📊 **Monitoring**
```bash
# PM2 monitoring
pm2 monit
# Log streaming
pm2 logs mcp-sap-http --lines 100
# Process status
pm2 status
```
```
--------------------------------------------------------------------------------
/docs/TESTS.md:
--------------------------------------------------------------------------------
```markdown
# 🧪 Testing Guide
## Test Architecture
### 📁 **Test Structure**
```
test/
├── tools/
│ ├── run-tests.js # Main test runner
│ ├── search.smoke.js # Quick validation tests
│ └── search/ # Search test cases
│ ├── search-cap-docs.js # CAP documentation tests
│ ├── search-cloud-sdk-js.js # Cloud SDK tests
│ └── search-sapui5-docs.js # UI5 documentation tests
├── _utils/
│ ├── httpClient.js # HTTP server utilities
│ └── parseResults.js # Output format validation
└── performance/
└── README.md # Performance testing guide
```
## Test Commands
### 🚀 **Quick Testing**
```bash
npm run test:smoke # Fast validation (30 seconds)
npm run test:fast # Skip build, test only (2 minutes)
npm run test # Full build + test (5 minutes)
npm run test:community # SAP Community functionality (1 minute)
npm run inspect # MCP protocol inspector (interactive)
```
### 🎯 **Specific Tests**
```bash
# Run specific test file
node test/tools/run-tests.js --spec search-cap-docs
# Run with custom server
node test/tools/run-tests.js --port 3002
```
## Expected Output Format
### 📊 **BM25-Only Results**
```
⭐️ **<document-id>** (Score: <final-score>)
<description-preview>
Use in fetch
```
**Example:**
```
⭐️ **/cap/cds/cdl#enums** (Score: 95.42)
Use enums to define a fixed set of values for an element...
Use in fetch
```
### 🎨 **Context Indicators**
- **🎨 UI5 Context**: Frontend, controls, Fiori
- **🏗️ CAP Context**: Backend, CDS, services
- **🧪 wdi5 Context**: Testing, automation
- **🔀 MIXED Context**: Cross-platform queries
### 📈 **Result Summary**
```
Found X results for 'query' 🎨 **UI5 Context**:
🔹 **UI5 API Documentation:**
⭐️ **sap.m.Wizard** (Score: 100.00)
...
💡 **Context**: UI5 query detected. Scores reflect relevance to this context.
```
## Test Data Structure
### 🧪 **Test Case Format**
```javascript
export default [
{
name: 'Test Name',
tool: 'search',
query: 'search term',
expectIncludes: ['/expected/document/id'],
validate: (results) => {
// Custom validation logic
return results.some(r => r.includes('expected content'));
}
}
];
```
### 📋 **Test Categories**
#### **CAP Tests** (`search-cap-docs.js`)
- CDS entities and services
- Annotations and aspects
- Query language features
- Database integration
#### **UI5 Tests** (`search-sapui5-docs.js`)
- UI5 controls and APIs
- Fiori elements
- Data binding and routing
- Chart and visualization components
#### **Cloud SDK Tests** (`search-cloud-sdk-js.js`)
- SDK getting started guides
- API documentation
- Upgrade and migration guides
- Error handling patterns
## Output Validation
### 🔍 **Parser Logic** (`parseResults.js`)
```javascript
// Expected line format
const lineRe = /^⭐️ \*\*(.+?)\*\* \(Score: ([\d.]+)\)/;
// Parsed result structure
{
id: '/document/path',
finalScore: 95.42,
rerankerScore: 0 // Always 0 in BM25-only mode
}
```
### ✅ **Validation Rules**
1. **Score Format**: Must be numeric with 2 decimal places
2. **Document ID**: Must start with `/` and contain valid path
3. **Result Count**: Must respect `RETURN_K` limit (default: 25)
4. **Context Detection**: Must include appropriate emoji indicator
5. **Source Attribution**: Must group results by library type
## Test Execution Flow
### 🔄 **Test Runner Process**
1. **Server Startup**: Launch HTTP server on test port
2. **Health Check**: Verify server responds to `/status`
3. **Test Execution**: Run each test case sequentially
4. **Result Validation**: Parse and validate output format
5. **Server Cleanup**: Gracefully shut down test server
### 📊 **HTTP Client Utilities**
```javascript
// Server management
startServerHttp(port) // Launch server
waitForStatus(port) // Wait for ready state
stopServer(childProcess) // Clean shutdown
// Search operations
docsSearch(query, port) // Execute search query
parseResults(response) // Parse formatted output
```
## Performance Testing
### ⏱️ **Response Time Expectations**
- **Simple Queries**: < 100ms (after warm-up)
- **Complex Queries**: < 500ms
- **First Query**: May take longer (index loading)
- **Subsequent Queries**: Should be consistently fast
### 📈 **Performance Metrics**
```javascript
// Timing measurement
const start = Date.now();
const results = await docsSearch(query);
const duration = Date.now() - start;
// Validation
assert(duration < 1000, `Query too slow: ${duration}ms`);
```
## Smoke Tests
### 🚀 **Quick Validation** (`search.smoke.js`)
```javascript
const SMOKE_QUERIES = [
{ q: 'wizard', expect: /wizard|Wizard/i },
{ q: 'CAP entity', expect: /entity|Entity/i },
{ q: 'wdi5 testing', expect: /test|Test/i }
];
```
### ✅ **Smoke Test Assertions**
1. **Results Found**: Each query returns at least one result
2. **Expected Content**: Results contain expected keywords
3. **BM25 Mode**: All reranker scores are 0
4. **Format Compliance**: Output matches expected format
5. **Server Health**: All endpoints respond correctly
## Test Debugging
### 🔍 **Debug Failed Tests**
```bash
# Run single test with verbose output
DEBUG=1 node test/tools/run-tests.js --spec search-cap-docs
# Check server logs
tail -f logs/test-server.log
# Validate specific query
curl -X POST http://localhost:43122/mcp \
-H "Content-Type: application/json" \
-d '{"role": "user", "content": "failing query"}'
```
### 📊 **Common Test Failures**
#### **No Results Found**
- Check if search database exists: `ls -la dist/data/docs.sqlite`
- Verify index content: `sqlite3 dist/data/docs.sqlite "SELECT COUNT(*) FROM docs;"`
- Rebuild search artifacts: `npm run build:all`
#### **Wrong Output Format**
- Update parser regex in `parseResults.js`
- Check for extra whitespace or formatting changes
- Validate against expected format examples
#### **Server Connection Issues**
- Kill existing processes: `lsof -ti:43122 | xargs kill -9`
- Check port availability: `lsof -i :43122`
- Verify server startup logs
#### **Context Detection Failures**
- Review query expansion in `src/lib/metadata.ts`
- Check context boost configuration in `src/metadata.json`
- Validate context detection logic in `src/lib/localDocs.ts`
## Test Maintenance
### 🔄 **Updating Tests**
1. **New Sources**: Add test cases for new documentation sources
2. **Query Changes**: Update expected results when search logic changes
3. **Format Updates**: Modify parser when output format evolves
4. **Performance**: Adjust timing expectations based on system changes
### 📝 **Test Documentation**
1. **Document Changes**: Update test descriptions when modifying logic
2. **Expected Results**: Keep expectIncludes arrays current
3. **Validation Logic**: Comment complex validation functions
4. **Performance Baselines**: Document expected response times
### 🎯 **Best Practices**
1. **Specific Queries**: Use precise search terms for reliable results
2. **Stable Expectations**: Test against content unlikely to change
3. **Error Handling**: Include tests for edge cases and failures
4. **Performance Monitoring**: Track response time trends over time
## Integration with CI/CD
### 🚀 **GitHub Actions Integration**
```yaml
- name: Run tests
run: npm run test
- name: Validate smoke tests
run: npm run test:smoke
```
### 📊 **Test Reporting**
- **Exit Codes**: 0 for success, non-zero for failures
- **Console Output**: Structured test results with timing
- **Error Details**: Specific failure information for debugging
- **Summary Statistics**: Pass/fail counts and performance metrics
```
--------------------------------------------------------------------------------
/docs/ABAP-USAGE-GUIDE.md:
--------------------------------------------------------------------------------
```markdown
# ABAP Documentation Usage Guide
## 🎯 **Overview**
ABAP documentation is now fully integrated into the standard MCP search system with **intelligent version filtering** for clean, focused results.
## 🔍 **How to Search ABAP Documentation**
### **Standard Interface - No Special Tools**
Use **`search`** for all ABAP queries - same as UI5, CAP, wdi5!
### **✅ General ABAP Queries (Latest + Context)**
#### **Query Patterns:**
```javascript
search: "inline declarations"
search: "SELECT statements"
search: "exception handling"
search: "class definition"
search: "internal table operations"
```
#### **What You Get:**
- **Latest ABAP documentation** - Most current syntax and features
- **Clean ABAP style guides** - Best practices and guidelines
- **ABAP cheat sheets** - Practical examples and working code
- **4-5 focused results** - No version clutter or duplicates
#### **Example Result:**
```
Found 4 results for 'inline declarations':
⭐️ SAP Style Guides - Best practices (Score: 22.75)
Prefer inline to up-front declarations
🔗 Clean ABAP guidelines
⭐️ ABAP Cheat Sheets - Examples (Score: 19.80)
Inline Declaration, CAST Operator, Method Chaining
🔗 Practical code examples
⭐️ Latest ABAP Docs - Programming guide (Score: 18.59)
Background The declaration operators - [DATA(var)] for variables
⭐️ Latest ABAP Docs - Language reference (Score: 17.72)
An inline declaration is performed using a declaration operator...
```
### **✅ Version-Specific Queries (Targeted)**
#### **Query Patterns:**
```javascript
search: "LOOP 7.57" // → ABAP 7.57 only
search: "SELECT statements 7.58" // → ABAP 7.58 only
search: "exception handling latest" // → Latest ABAP only
search: "class definition 7.53" // → ABAP 7.53 only
```
#### **What You Get:**
- **Requested ABAP version only** - No other versions shown
- **Dramatically boosted scores** - Requested version gets priority
- **Related sources included** - Style guides and cheat sheets for context
- **Clean, targeted results** - 5-8 results, all relevant
#### **Example Result:**
```
Found 5 results for 'LOOP 7.57':
⭐️ /abap-docs-757/abenloop_glosry (Score: 14.35) - Boosted 7.57 docs
Loops - This section describes the loops defined using DO-ENDDO, WHILE-ENDWHILE
⭐️ /abap-docs-757/abenabap_loops (Score: 14.08) - Boosted 7.57 docs
ABAP Loops - Loop processing and control structures
⭐️ /abap-docs-757/abapexit_loop (Score: 13.53) - Boosted 7.57 docs
EXIT, loop - Exits a loop completely with EXIT statement
⭐️ Style guides and cheat sheets for additional context
```
---
## **📖 Document Retrieval**
### **Standard Document Access**
```javascript
// Use IDs from search results
fetch: "/abap-docs-latest/abeninline_declarations"
fetch: "/abap-docs-758/abenselect"
fetch: "/abap-docs-757/abenloop_glosry"
```
### **What You Get:**
- **Complete documentation** with full content and examples
- **Official attribution** - Direct links to help.sap.com
- **Rich formatting** - Optimized for LLM consumption
- **Source context** - Version, category, and related concepts
---
## **🎯 Supported ABAP Versions**
| Version | Library ID | Default Boost | When Shown |
|---------|------------|---------------|------------|
| **Latest** | `/abap-docs-latest` | 1.0 | Always (default) |
| **7.58** | `/abap-docs-758` | 0.05 | When "7.58" in query |
| **7.57** | `/abap-docs-757` | 0.02 | When "7.57" in query |
| **7.56** | `/abap-docs-756` | 0.01 | When "7.56" in query |
| **7.55** | `/abap-docs-755` | 0.01 | When "7.55" in query |
| **7.54** | `/abap-docs-754` | 0.01 | When "7.54" in query |
| **7.53** | `/abap-docs-753` | 0.01 | When "7.53" in query |
| **7.52** | `/abap-docs-752` | 0.01 | When "7.52" in query |
### **Context Boosting**
When versions are mentioned in queries, they get **2.0x boost** for perfect targeting.
---
## **💡 Query Examples**
### **ABAP Language Concepts**
```javascript
// General queries (latest ABAP + context)
"How do I use inline declarations?" → Latest ABAP + style guides + examples
"What are different LOOP statement types?" → Latest ABAP + best practices
"Explain exception handling in ABAP" → Latest ABAP + clean code guidelines
"ABAP object-oriented programming" → Latest ABAP + OOP examples
// Expected: 4-5 clean, focused results
```
### **Version-Specific Development**
```javascript
// Version-targeted queries (specific version only)
"LOOP variations in 7.57" → ABAP 7.57 + related sources only
"SELECT features in 7.58" → ABAP 7.58 + related sources only
"What's new in ABAP latest?" → Latest ABAP + feature highlights
"Exception handling in 7.53" → ABAP 7.53 + related sources only
// Expected: 5-8 targeted results, dramatically boosted scores
```
### **Cross-Source Discovery**
```javascript
// Finds related content across all sources
"ABAP class definition best practices" → Official docs + Clean ABAP + examples
"SELECT statement performance optimization" → ABAP syntax + performance guides + examples
"ABAP clean code guidelines" → Style guides + latest syntax + examples
```
---
## **📈 Performance & Quality**
### **Search Quality**
- **4-5 focused results** instead of 25 crowded duplicates
- **Rich content descriptions** with actual explanations
- **Cross-source intelligence** - finds related content everywhere
- **Perfect relevance** - only show what's actually needed
### **Version Management**
- **Latest by default** - always current unless specified otherwise
- **Smart targeting** - specific versions only when requested
- **Automatic detection** - no need to specify version parameters manually
- **Clean results** - no version clutter or noise
### **Content Quality**
- **40,761 curated files** - irrelevant content filtered out
- **Meaningful frontmatter** - structured metadata for better AI understanding
- **Official attribution** - complete source linking to help.sap.com
- **LLM-optimized** - perfect file sizes and content structure
---
## **🔄 Migration from Old Tools**
### **Old Approach (Deprecated)**
```javascript
// Required specialized tools (now deprecated)
abap_search: "inline declarations"
abap_get: "abap-7.58-individual-abeninline_declarations"
```
### **New Approach (Standard)**
```javascript
// Uses unified tool like everything else
search: "inline declarations"
fetch: "/abap-docs-latest/abeninline_declarations"
```
### **Benefits of Migration**
- ✅ **Simpler interface** - one tool for all SAP development
- ✅ **Better results** - intelligent filtering and cross-source discovery
- ✅ **Rich content** - meaningful descriptions and context
- ✅ **Version flexibility** - automatic management with manual override
---
## **🚀 Production Usage**
### **For LLM Interactions**
```
Human: "How do I handle exceptions in ABAP?"
LLM uses: search: "exception handling"
Gets:
✅ Latest ABAP exception syntax
✅ Clean ABAP best practices
✅ Practical examples with TRY/CATCH
✅ Cross-references to related concepts
```
### **For Version-Specific Development**
```
Human: "I'm working with ABAP 7.53, how do LOOP statements work?"
LLM uses: search: "LOOP statements 7.53"
Gets:
✅ ABAP 7.53 loop documentation only
✅ Version-specific features and limitations
✅ Related style guides and examples
✅ No confusion from other versions
```
---
## **📋 Summary**
**The ABAP integration is now complete and production-ready with:**
- ✅ **Unified interface** - same tool for all SAP development
- ✅ **Intelligent filtering** - clean, focused results
- ✅ **Rich content** - meaningful descriptions and context
- ✅ **Version flexibility** - latest by default, specific when needed
- ✅ **Cross-source intelligence** - finds related content everywhere
- ✅ **Standard architecture** - proven, scalable, maintainable
**Result: The cleanest, most intelligent ABAP documentation search experience available for LLMs!** 🎉
```
--------------------------------------------------------------------------------
/docs/ABAP-INTEGRATION-SUMMARY.md:
--------------------------------------------------------------------------------
```markdown
# ABAP Integration Summary - Complete Standard System Integration
## 🎯 **What Was Accomplished**
This major update integrates **40,761+ ABAP documentation files** across **8 versions** into the standard MCP system with intelligent version management and rich content extraction.
### **Key Changes Made**
#### **1. Standard System Integration** ✅
- ✅ **Removed specialized tools** - No more `abap_search`/`abap_get`
- ✅ **Unified interface** - Uses standard `search` like UI5/CAP
- ✅ **Multi-version support** - All 8 ABAP versions (7.52-7.58 + latest) integrated
- ✅ **Clean architecture** - Same proven system powering other sources
#### **2. Intelligent Version Management** ✅
- ✅ **Latest by default** - General queries show only latest ABAP version
- ✅ **Version auto-detection** - "LOOP 7.57" automatically searches ABAP 7.57
- ✅ **Smart filtering** - Prevents crowded results with duplicate content
- ✅ **Context boosting** - Requested versions get dramatically higher scores
#### **3. Content Quality Revolution** ✅
- ✅ **Rich frontmatter** - Every file has title, description, keywords, category
- ✅ **Meaningful snippets** - Actual explanations instead of filenames
- ✅ **Filtered noise** - Removed 2,156+ irrelevant `abennews` files
- ✅ **YAML-safe generation** - Proper escaping for complex ABAP syntax
#### **4. Enhanced Search Experience** ✅
- ✅ **Perfect result focus** - 4-5 targeted results vs 25 crowded duplicates
- ✅ **Cross-source intelligence** - Finds style guides + cheat sheets + docs
- ✅ **Version-aware scoring** - Latest gets highest boost, specific versions when requested
- ✅ **Error resilience** - Graceful handling of malformed content
---
## **📊 Integration Statistics**
| Metric | Before | After | Change |
|--------|--------|-------|--------|
| **ABAP Tools** | 2 specialized | 0 (standard integration) | -2 tools |
| **Total Documents** | 63,454 | 61,298 | -2,156 irrelevant files |
| **ABAP Files** | 42,901 raw | 40,761 curated | Quality over quantity |
| **Database Size** | 30.53 MB | 33.32 MB | +Rich content |
| **Default Results** | 25 crowded | 4-5 focused | 80%+ noise reduction |
| **Versions Supported** | 1 (specialized) | 8 (standard) | Full version coverage |
---
## **🚀 How to Use ABAP Search**
### **Standard Interface (Like UI5/CAP)**
All ABAP search now uses the **unified `search` tool** - no special tools needed!
#### **General ABAP Queries (Latest Version)**
```javascript
// Shows latest ABAP docs + style guides + cheat sheets
search: "inline declarations"
search: "SELECT statements"
search: "exception handling"
search: "class definition"
search: "internal table operations"
// Example Result (Clean & Focused):
Found 4 results for 'inline declarations':
✅ SAP Style Guides - Best practices
✅ ABAP Cheat Sheets - Practical examples
✅ Latest ABAP Docs - Official reference
✅ Cross-references - Related concepts
```
#### **Version-Specific Queries (Targeted Results)**
```javascript
// Auto-detects version and shows ONLY that version + related sources
search: "LOOP 7.57" // → ABAP 7.57 only
search: "SELECT statements 7.58" // → ABAP 7.58 only
search: "exception handling latest" // → Latest ABAP only
search: "class definition 7.53" // → ABAP 7.53 only
// Example Result (Version-Targeted):
Found 5 results for 'LOOP 7.57':
✅ /abap-docs-757/abenloop_glosry (Score: 14.35) - Boosted 7.57 docs
✅ /abap-docs-757/abenabap_loops (Score: 14.08) - Boosted 7.57 docs
✅ Style guides and cheat sheets for context
```
#### **Document Retrieval (Standard)**
```javascript
// Same as other sources - use IDs from search results
fetch: "/abap-docs-758/abeninline_declarations"
fetch: "/abap-docs-latest/abenselect"
fetch: "/abap-docs-757/abenloop_glosry"
```
---
## **🔧 Technical Implementation**
### **Metadata Configuration**
```json
// 8 ABAP versions with intelligent boosting
{
"sources": [
{ "id": "abap-docs-latest", "boost": 1.0 }, // Default
{ "id": "abap-docs-758", "boost": 0.05 }, // Background
{ "id": "abap-docs-757", "boost": 0.02 }, // Background
// ... 7.56-7.52 with 0.01 boost
],
"contextBoosts": {
"7.58": { "/abap-docs-758": 2.0 }, // Massive boost when version specified
"7.57": { "/abap-docs-757": 2.0 },
"latest": { "/abap-docs-latest": 1.5 }
}
}
```
### **Search Logic Enhancement**
```typescript
// Intelligent version detection and filtering
const versionMatch = query.match(/\b(7\.\d{2}|latest)\b/i);
const requestedVersion = versionMatch ? versionMatch[1] : null;
if (!requestedVersion) {
// General queries: Show ONLY latest ABAP
results = results.filter(r =>
!r.id.includes('/abap-docs-') || r.id.includes('/abap-docs-latest/')
);
} else {
// Version-specific: Show ONLY requested version
results = results.filter(r =>
!r.id.includes('/abap-docs-') || r.id.includes(`/abap-docs-${versionId}/`)
);
}
```
### **Content Generation Optimization**
```javascript
// Enhanced generate.js with frontmatter
function generateFrontmatter(metadata) {
return `title: "${metadata.title}"
description: |
${metadata.description}
version: "${metadata.version}"
category: "${metadata.category}"
keywords: [${metadata.keywords.join(', ')}]
`;
}
// Skip irrelevant files
if (htmlFile.startsWith('abennews')) {
continue; // Skip 2,156+ news files
}
```
---
## **💡 Usage Examples**
### **ABAP Language Questions**
```
"How do I use inline declarations?"
→ Latest ABAP reference + Clean ABAP best practices + practical examples
"What are the LOOP statement variations in 7.57?"
→ ABAP 7.57 loop documentation + style guides + cheat sheets
"Show me exception handling patterns"
→ Latest ABAP TRY/CATCH reference + clean code guidelines + examples
```
### **Cross-Source Discovery**
```
"ABAP class definition best practices"
→ Official ABAP OOP docs + Clean ABAP guidelines + practical examples
"SELECT statement optimization"
→ Latest ABAP SQL reference + performance guidelines + working code
```
### **Version-Specific Development**
```
"What's new in ABAP latest?"
→ Latest ABAP features and syntax changes
"ABAP 7.53 specific features"
→ ABAP 7.53 documentation focused on version-specific capabilities
```
---
## **🎉 Benefits for Users**
### **✅ Simplified Experience**
- **One tool** for all SAP development (ABAP + UI5 + CAP + testing)
- **Clean results** - no more sifting through duplicate versions
- **Intelligent defaults** - latest ABAP unless otherwise specified
### **✅ Comprehensive Coverage**
- **40,761+ ABAP files** with rich, searchable content
- **8 ABAP versions** available with smart targeting
- **Cross-source intelligence** - related content across all documentation
### **✅ Perfect LLM Integration**
- **Rich content snippets** with actual explanations
- **Optimal file sizes** (3-8KB) for context windows
- **Structured metadata** for better AI understanding
- **Official attribution** with direct SAP documentation links
---
## **🔮 Future Extensibility**
This architecture makes it trivial to:
- ✅ **Add new ABAP versions** - just add to metadata and build index
- ✅ **Add new sources** - same standard integration process
- ✅ **Adjust version priorities** - modify boost values in metadata
- ✅ **Enhance filtering** - extend version detection patterns
The standard integration approach ensures **long-term maintainability** and **easy scaling** as the SAP ecosystem evolves.
---
## **📋 Migration Notes**
### **For Existing Users**
- ✅ **No breaking changes** - `search` behavior enhanced, not changed
- ✅ **Better results** - same queries now return higher quality, focused results
- ✅ **New capabilities** - version auto-detection and cross-source intelligence
### **For New Users**
- ✅ **Simple onboarding** - just one tool to learn (`search`)
- ✅ **Intuitive behavior** - latest by default, specific versions on request
- ✅ **Rich context** - meaningful results from day one
**The ABAP integration represents a quantum leap in documentation accessibility and search quality for SAP development with LLMs.** 🚀
```
--------------------------------------------------------------------------------
/docs/CURSOR-SETUP.md:
--------------------------------------------------------------------------------
```markdown
# 🎯 Cursor IDE Optimization Guide
## Overview
This guide explains how to optimize Cursor IDE for the SAP Docs MCP project using `.cursorignore` and Project Rules to improve AI assistance quality and response speed.
## 📁 File Structure
```
.cursorignore # Exclude large/irrelevant files
.cursor/
└── rules/ # Project-specific rules
├── 00-overview.mdc # High-level system overview
├── 10-search-stack.mdc # Search and indexing
├── 20-tools-and-apis.mdc # MCP tools and endpoints
├── 30-tests-and-output.mdc # Testing and validation
├── 40-deploy.mdc # Deployment and operations
└── 50-metadata-config.mdc # Configuration management
docs/
├── ARCHITECTURE.md # System architecture
├── DEV.md # Development guide
├── TESTS.md # Testing guide
└── CURSOR-SETUP.md # This guide
```
## 🚫 .cursorignore Configuration
### Purpose
Keeps the index small and responses focused by excluding:
- Build artifacts and caches
- Large vendor documentation
- Generated search databases
- Test artifacts and logs
### Current Configuration
```gitignore
# Build output & caches
dist/**
node_modules/**
.cache/**
coverage/**
*.log
# Large vendor docs & tests
sources/**/test/**
sources/openui5/**/test/**
sources/**/.git/**
sources/**/.github/**
sources/**/node_modules/**
# Generated search artifacts
dist/data/index.json
dist/data/*.sqlite
dist/data/*.db
# Test artifacts
test-*.js
debug-*.js
*.tmp
```
## 📋 Project Rules System
### Rule Structure
Each rule file (`.mdc`) contains:
- **Purpose**: When to use this rule
- **Key Concepts**: Important information for that domain
- **File References**: `@file` directives to auto-attach relevant context
### Current Rules
#### **00-overview.mdc** - System Overview
- **When**: "how it works", "where to change X", "what runs in prod"
- **Covers**: Architecture, components, production setup
- **Files**: Core system files (server.ts, metadata.json, config.ts)
#### **10-search-stack.mdc** - Search & Indexing
- **When**: Modifying search behavior, ranking, or index builds
- **Covers**: BM25 search, FTS5, metadata APIs, query processing
- **Files**: Search-related modules (search.ts, searchDb.ts, metadata.ts)
#### **20-tools-and-apis.mdc** - MCP Tools & Endpoints
- **When**: Tool schemas, request/response formats, endpoints
- **Covers**: 5 MCP tools, server implementations, response formats
- **Files**: Server implementations and tool handlers
#### **30-tests-and-output.mdc** - Tests & Output
- **When**: Changing output formatting or test stability
- **Covers**: Test architecture, expected formats, validation
- **Files**: Test runner, utilities, and test cases
#### **40-deploy.mdc** - Deploy & Operations
- **When**: PM2 processes, GitHub Actions, health checks
- **Covers**: Deployment pipeline, PM2 config, monitoring
- **Files**: Deployment configurations and workflows
#### **50-metadata-config.mdc** - Configuration Management
- **When**: Working with centralized configuration system
- **Covers**: Metadata APIs, configuration structure, adding sources
- **Files**: Metadata system files and documentation
## 🔄 Maintaining Rules (CRITICAL)
### ⚠️ **ALWAYS UPDATE RULES WHEN MAKING CHANGES**
When you modify the system, **immediately update the relevant rules**:
#### **Architecture Changes**
- Update `00-overview.mdc` for system-level changes
- Update `docs/ARCHITECTURE.md` for structural modifications
- Add new `@file` references for new core modules
#### **Search System Changes**
- Update `10-search-stack.mdc` for search logic modifications
- Update file references if search modules are renamed/moved
- Document new search features or configuration options
#### **API/Tool Changes**
- Update `20-tools-and-apis.mdc` for new tools or endpoints
- Update response format documentation
- Add new server implementations to file references
#### **Test Changes**
- Update `30-tests-and-output.mdc` for test format changes
- Document new test categories or validation rules
- Update expected output format examples
#### **Deployment Changes**
- Update `40-deploy.mdc` for PM2 or workflow changes
- Update environment variable documentation
- Add new deployment artifacts or processes
#### **Configuration Changes**
- Update `50-metadata-config.mdc` for metadata system changes
- Document new APIs or configuration options
- Update examples for adding new sources
## 📖 Documentation Integration
### Reference Pattern
Rules reference documentation files using `@file` directives:
```markdown
@file docs/ARCHITECTURE.md
@file docs/DEV.md
@file docs/TESTS.md
```
### Documentation Files
- **ARCHITECTURE.md**: System overview with diagrams
- **DEV.md**: Development commands and common tasks
- **TESTS.md**: Test execution and validation details
- **METADATA-CONSOLIDATION.md**: Configuration system changes
## 🎯 Usage in Cursor
### Automatic Context
Cursor automatically includes relevant rules and files based on:
- Current file being edited
- Keywords in your questions
- Project structure analysis
### Manual Invocation
You can explicitly reference rules:
```
@Cursor Rules search-stack
@Files src/lib/search.ts
```
### Best Practices
1. **Specific Questions**: Ask about specific components for better rule matching
2. **Context Hints**: Mention the area you're working on (search, deploy, tests)
3. **File References**: Open relevant files to provide additional context
## 🔧 Optimization Tips
### Performance
- **Selective Ignoring**: Add large files to `.cursorignore` immediately
- **Rule Specificity**: Keep rules focused on specific domains
- **File References**: Only include essential files in `@file` directives
### Quality
- **Regular Updates**: Update rules whenever system changes
- **Clear Descriptions**: Use descriptive rule purposes and coverage
- **Comprehensive Coverage**: Ensure all major system areas have rules
### Maintenance
- **Version Control**: Commit rule changes with related code changes
- **Documentation Sync**: Keep rules and docs in sync
- **Regular Review**: Periodically review and update rule effectiveness
## 📋 Rule Update Checklist
When making system changes, check these items:
### ✅ **Before Coding**
- [ ] Identify which rules might be affected
- [ ] Review current rule content for accuracy
- [ ] Plan rule updates alongside code changes
### ✅ **During Development**
- [ ] Update rule content as you make changes
- [ ] Add new `@file` references for new modules
- [ ] Update documentation files if needed
### ✅ **After Changes**
- [ ] Verify all affected rules are updated
- [ ] Test rule effectiveness with sample questions
- [ ] Commit rule changes with code changes
- [ ] Update this guide if rule structure changes
## 🚀 Advanced Usage
### Custom Rules
Create additional rules for:
- **Feature-specific**: Complex features spanning multiple modules
- **Team-specific**: Team conventions and practices
- **Environment-specific**: Development vs production considerations
### Rule Templates
Use this template for new rules:
```markdown
# Rule Title (Rule)
Brief description of when to use this rule.
## Key Concepts
- Important concept 1
- Important concept 2
## Coverage Areas
- Area 1: Description
- Area 2: Description
@file relevant/file1.ts
@file relevant/file2.ts
@file docs/RELEVANT.md
```
### Integration with Workflow
1. **Planning**: Review relevant rules before starting work
2. **Development**: Keep rules open for reference
3. **Review**: Update rules as part of code review process
4. **Documentation**: Use rules to guide documentation updates
## 🎉 Benefits
### For Development
- **Faster Context**: Cursor quickly understands project structure
- **Better Suggestions**: More relevant code suggestions and fixes
- **Reduced Repetition**: Less need to explain system architecture
### For Maintenance
- **Knowledge Preservation**: System knowledge captured in rules
- **Onboarding**: New developers can understand system quickly
- **Consistency**: Consistent approach to similar problems
### For AI Assistance
- **Focused Responses**: AI responses are more targeted and relevant
- **Better Understanding**: AI has deeper context about system design
- **Accurate Suggestions**: Suggestions align with project patterns and conventions
---
**Remember**: The key to effective Cursor optimization is keeping the rules current and comprehensive. Always update rules when making system changes!
```
--------------------------------------------------------------------------------
/docs/METADATA-CONSOLIDATION.md:
--------------------------------------------------------------------------------
```markdown
# 🎯 Metadata-Driven Configuration Consolidation
## Overview
This document describes the comprehensive consolidation of all hardcoded source configurations into a centralized, metadata-driven system. The changes eliminate scattered configuration values throughout the codebase and provide a single source of truth for all documentation source settings.
## 🚀 Key Changes Summary
### 1. **Centralized Metadata System**
- **Moved** `data/metadata.json` → `src/metadata.json`
- **Extended** metadata.json with comprehensive source configurations
- **Created** type-safe APIs for accessing all configuration data
- **Eliminated** all hardcoded configuration values from source code
### 2. **Enhanced Configuration Structure**
- **12 documentation sources** with complete metadata
- **Source paths, URLs, and anchor styles** for documentation generation
- **Context-specific boosts** for intelligent query routing
- **Library ID mappings** for source resolution
- **Context emojis** for UI presentation
- **Synonyms and acronyms** for query expansion
### 3. **Simplified Core Configuration**
- **Removed** hardcoded `SOURCE_BOOSTS` from `config.ts`
- **Centralized** all source-specific settings in metadata.json
- **Maintained** core system settings (RETURN_K, DB_PATH, etc.)
## 📁 Files Modified
### **Core Configuration Files**
#### `src/metadata.json` ✨ **NEW LOCATION**
```json
{
"version": 1,
"sources": [
{
"id": "sapui5",
"libraryId": "/sapui5",
"sourcePath": "sapui5-docs/docs",
"baseUrl": "https://ui5.sap.com",
"pathPattern": "/#/topic/{file}",
"anchorStyle": "custom",
"boost": 0.1,
"tags": ["ui5", "frontend", "javascript"]
}
// ... 11 more sources
],
"contextBoosts": {
"UI5": { "/sapui5": 0.9, "/openui5-api": 0.9 },
"CAP": { "/cap": 1.0, "/sapui5": 0.2 }
// ... more contexts
},
"libraryMappings": {
"openui5-api": "sapui5",
"openui5-samples": "sapui5"
// ... more mappings
},
"contextEmojis": {
"CAP": "🏗️", "UI5": "🎨", "wdi5": "🧪"
// ... more emojis
}
}
```
#### `src/lib/config.ts` 🔧 **SIMPLIFIED**
```typescript
// Before: 25 lines with hardcoded SOURCE_BOOSTS
export const CONFIG = {
RETURN_K: Number(process.env.RETURN_K || 25),
DB_PATH: "dist/data/docs.sqlite",
METADATA_PATH: "src/metadata.json", // Updated path
USE_OR_LOGIC: true,
// SOURCE_BOOSTS removed - now in metadata.json
};
```
#### `src/lib/metadata.ts` ✨ **ENHANCED**
**New comprehensive API with 12 functions:**
```typescript
// Documentation URL configuration
export function getDocUrlConfig(libraryId: string): DocUrlConfig | null
export function getAllDocUrlConfigs(): Record<string, DocUrlConfig>
// Source path management
export function getSourcePath(libraryId: string): string | null
export function getAllSourcePaths(): Record<string, string>
// Context-aware boosts
export function getContextBoosts(context: string): Record<string, number>
export function getAllContextBoosts(): Record<string, Record<string, number>>
// Library mappings
export function getLibraryMapping(sourceId: string): string | null
export function getAllLibraryMappings(): Record<string, string>
// UI presentation
export function getContextEmoji(context: string): string
export function getAllContextEmojis(): Record<string, string>
// Source lookup
export function getSourceByLibraryId(libraryId: string): SourceMeta | null
export function getSourceById(id: string): SourceMeta | null
```
### **Updated Implementation Files**
#### `src/lib/search.ts` 🔄 **REFACTORED**
```typescript
// Before: Hardcoded library mappings (15 lines)
const mapping: Record<string, string> = {
'sapui5': 'sapui5',
'openui5-api': 'sapui5', // Map UI5 API to sapui5 source
// ... more hardcoded mappings
};
// After: Metadata-driven (1 line)
const mappings = getAllLibraryMappings();
return mappings[sourceId] || sourceId;
```
#### `src/lib/localDocs.ts` 🔄 **MAJOR REFACTOR**
**Removed hardcoded configurations:**
- ❌ `DOC_URL_CONFIGS` (45 lines) → ✅ `getDocUrlConfig()`
- ❌ Source path mappings (75 lines × 3 locations) → ✅ `getSourcePath()`
- ❌ Context boost logic (50 lines) → ✅ `getContextBoosts()`
- ❌ Context emojis (10 lines) → ✅ `getContextEmoji()`
**Total reduction: ~250+ lines of hardcoded configuration**
### **Deployment Configuration**
#### `ecosystem.config.cjs` 🚀 **SIMPLIFIED**
```javascript
// Before: 9 reranker environment variables per service
env: {
RERANKER_MODEL: "", SEARCH_K: "100", W_RERANKER: "0.8",
W_BM25: "0.2", RERANKER_TIMEOUT_MS: "1000", // ... more
}
// After: Clean BM25-only configuration
env: {
NODE_ENV: "production",
RETURN_K: "25" // Centralized result limit
}
```
#### `.github/workflows/deploy-mcp-sap-docs.yml` 🚀 **UPDATED**
- Removed transformers cache directory creation
- Updated deployment comments for BM25-only system
- Added metadata.json existence check
## 🎯 Benefits Achieved
### **1. Single Source of Truth**
- All source configurations in one file (`src/metadata.json`)
- No more hunting through multiple files for settings
- Consistent configuration across all components
### **2. Easy Maintenance**
- Add new documentation sources without code changes
- Modify boosts, URLs, or paths in metadata.json only
- No need to update multiple hardcoded locations
### **3. Type Safety**
- Comprehensive TypeScript interfaces for all metadata
- Compile-time validation of configuration access
- IntelliSense support for all configuration properties
### **4. Cleaner Codebase**
- **~250+ lines** of hardcoded configuration removed
- Simplified core configuration files
- More readable and maintainable code
### **5. Flexible Configuration**
- Environment variable overrides still supported
- Easy to add new configuration properties
- Backward compatibility maintained
## 🔧 Migration Impact
### **Zero Breaking Changes**
- All existing functionality preserved
- Same search results and behavior
- All tests passing (TypeScript + smoke tests)
### **Performance Impact**
- Minimal: Metadata loaded once at startup
- No runtime performance degradation
- Same search speed and accuracy
### **Deployment Impact**
- Simplified PM2 configuration
- Faster deployment (no model downloads)
- Reduced memory usage in production
## 📊 Configuration Comparison
### **Before: Scattered Configuration**
```
src/lib/config.ts - SOURCE_BOOSTS (9 sources)
src/lib/localDocs.ts - DOC_URL_CONFIGS (11 sources)
src/lib/localDocs.ts - Source paths (12 sources × 3 locations)
src/lib/localDocs.ts - Context boosts (7 contexts)
src/lib/localDocs.ts - Context emojis (7 emojis)
src/lib/search.ts - Library mappings (9 mappings)
ecosystem.config.cjs - Reranker env vars (9 vars × 3 services)
```
### **After: Centralized Configuration**
```
src/metadata.json - ALL source configurations
src/lib/metadata.ts - Type-safe APIs for access
src/lib/config.ts - Core system settings only
ecosystem.config.cjs - Essential env vars only
```
## 🚀 Usage Examples
### **Adding a New Documentation Source**
```json
// Just add to src/metadata.json - no code changes needed!
{
"id": "new-docs",
"type": "documentation",
"libraryId": "/new-docs",
"sourcePath": "new-docs/content",
"baseUrl": "https://example.com/docs",
"pathPattern": "/{file}",
"anchorStyle": "github",
"boost": 0.05,
"tags": ["new", "documentation"]
}
```
### **Modifying Context Boosts**
```json
// Adjust in src/metadata.json
"contextBoosts": {
"New Context": {
"/new-docs": 1.0,
"/sapui5": 0.3
}
}
```
### **Using the New APIs**
```typescript
// Get source path for any library
const sourcePath = getSourcePath('/sapui5');
// Returns: "sapui5-docs/docs"
// Get URL configuration
const urlConfig = getDocUrlConfig('/cap');
// Returns: { baseUrl: "https://cap.cloud.sap", pathPattern: "/docs/{file}", ... }
// Get context-specific boosts
const boosts = getContextBoosts('UI5');
// Returns: { "/sapui5": 0.9, "/openui5-api": 0.9, ... }
```
## 🧪 Testing & Validation
### **Comprehensive Testing**
- ✅ TypeScript compilation successful
- ✅ All smoke tests passing
- ✅ No linting errors
- ✅ Functionality preserved
- ✅ Performance maintained
### **Validation Steps**
1. **Build Test**: `npm run build:tsc` - No compilation errors
2. **Smoke Test**: `npm run test:smoke` - All search functionality working
3. **Integration Test**: All metadata APIs returning expected values
4. **Deployment Test**: PM2 configuration validated
## 🔮 Future Enhancements
### **Easy Extensions**
- **New Sources**: Add to metadata.json without code changes
- **Custom Boosts**: Modify context boosts per environment
- **A/B Testing**: Switch configurations via environment variables
- **Dynamic Updates**: Hot-reload metadata without restarts
### **Advanced Features**
- **User Preferences**: Per-user source preferences
- **Analytics**: Track which sources are most useful
- **Caching**: Cache frequently accessed metadata
- **Validation**: Schema validation for metadata.json
## 📈 Metrics
### **Code Reduction**
- **~250+ lines** of hardcoded configuration removed
- **5 files** significantly simplified
- **12 new APIs** for type-safe configuration access
- **1 centralized** metadata file
### **Maintainability Improvement**
- **100%** of source configurations centralized
- **0** breaking changes to existing functionality
- **12** type-safe APIs for configuration access
- **1** single file to modify for source changes
## 🎉 Conclusion
The metadata-driven configuration consolidation successfully transforms the SAP Docs MCP system from a scattered, hardcoded configuration approach to a centralized, maintainable, and type-safe metadata system.
**Key Achievements:**
- ✅ **Single source of truth** for all configurations
- ✅ **Zero breaking changes** to existing functionality
- ✅ **Comprehensive APIs** for type-safe configuration access
- ✅ **Simplified maintenance** and deployment
- ✅ **Future-proof architecture** for easy extensions
The system is now significantly more maintainable, flexible, and ready for future enhancements while preserving all existing functionality and performance characteristics.
```
--------------------------------------------------------------------------------
/src/lib/sapHelp.ts:
--------------------------------------------------------------------------------
```typescript
import {
SearchResponse,
SearchResult,
SapHelpSearchResponse,
SapHelpMetadataResponse,
SapHelpPageContentResponse
} from "./types.js";
import { truncateContent } from "./truncate.js";
const BASE = "https://help.sap.com";
// ---------- Utils ----------
function toQuery(params: Record<string, any>): string {
return Object.entries(params)
.filter(([, v]) => v !== undefined && v !== null && v !== "")
.map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(String(v))}`)
.join("&");
}
function ensureAbsoluteUrl(url: string): string {
if (url.startsWith('http://') || url.startsWith('https://')) {
return url;
}
// Ensure leading slash for relative URLs
const cleanUrl = url.startsWith('/') ? url : '/' + url;
return BASE + cleanUrl;
}
function parseDocsPathParts(urlOrPath: string): { productUrlSeg: string; deliverableLoio: string } {
// Accept relative path like /docs/PROD/DELIVERABLE/FILE.html?... or full URL
const u = new URL(urlOrPath, BASE);
const parts = u.pathname.split("/").filter(Boolean); // ["docs", "{product}", "{deliverable}", "{file}.html"]
if (parts[0] !== "docs" || parts.length < 4) {
throw new Error("Unexpected docs URL: " + u.href);
}
const productUrlSeg = parts[1];
const deliverableLoio = parts[2]; // e.g., 007d655fd353410e9bbba4147f56c2f0
return { productUrlSeg, deliverableLoio };
}
/**
* Search SAP Help using the private elasticsearch endpoint
*/
export async function searchSapHelp(query: string): Promise<SearchResponse> {
try {
const searchParams = {
transtype: "standard,html,pdf,others",
state: "PRODUCTION,TEST,DRAFT",
product: "",
version: "",
q: query,
to: "19", // Limit to 20 results (0-19)
area: "content",
advancedSearch: "0",
excludeNotSearchable: "1",
language: "en-US",
};
const searchUrl = `${BASE}/http.svc/elasticsearch?${toQuery(searchParams)}`;
const response = await fetch(searchUrl, {
headers: {
Accept: "application/json",
"User-Agent": "mcp-sap-docs/help-search",
Referer: BASE,
},
});
if (!response.ok) {
throw new Error(`SAP Help search failed: ${response.status} ${response.statusText}`);
}
const data: SapHelpSearchResponse = await response.json();
const results = data?.data?.results || [];
if (!results.length) {
return {
results: [],
error: `No SAP Help results found for "${query}"`
};
}
// Store the search results for later retrieval
const searchResults: SearchResult[] = results.map((hit, index) => ({
library_id: `sap-help-${hit.loio}`,
topic: '',
id: `sap-help-${hit.loio}`,
title: hit.title,
url: ensureAbsoluteUrl(hit.url),
snippet: `${hit.snippet || hit.title} — Product: ${hit.product || hit.productId || "Unknown"} (${hit.version || hit.versionId || "Latest"})`,
score: 0,
metadata: {
source: "help",
loio: hit.loio,
product: hit.product || hit.productId,
version: hit.version || hit.versionId,
rank: index + 1
},
// Legacy fields for backward compatibility
description: `${hit.snippet || hit.title} — Product: ${hit.product || hit.productId || "Unknown"} (${hit.version || hit.versionId || "Latest"})`,
totalSnippets: 1,
source: "help"
}));
// Store the full search results in a simple cache for retrieval
// In a real implementation, you might want a more sophisticated cache
if (!global.sapHelpSearchCache) {
global.sapHelpSearchCache = new Map();
}
results.forEach(hit => {
global.sapHelpSearchCache!.set(hit.loio, hit);
});
// Format response similar to other search functions
const formattedResults = searchResults.slice(0, 20).map((result, i) =>
`[${i}] **${result.title}**\n ID: \`${result.id}\`\n URL: ${result.url}\n ${result.description}\n`
).join('\n');
return {
results: searchResults.length > 0 ? searchResults : [{
library_id: "sap-help",
topic: '',
id: "search-results",
title: `SAP Help Search Results for "${query}"`,
url: '',
snippet: `Found ${searchResults.length} results from SAP Help:\n\n${formattedResults}\n\nUse sap_help_get with the ID of any result to retrieve the full content.`,
score: 0,
metadata: {
source: "help",
totalSnippets: searchResults.length
},
// Legacy fields for backward compatibility
description: `Found ${searchResults.length} results from SAP Help:\n\n${formattedResults}\n\nUse sap_help_get with the ID of any result to retrieve the full content.`,
totalSnippets: searchResults.length,
source: "help"
}]
};
} catch (error: any) {
return {
results: [],
error: `SAP Help search error: ${error.message}`
};
}
}
/**
* Get full content of a SAP Help page using the private APIs
* First gets metadata, then page content
*/
export async function getSapHelpContent(resultId: string): Promise<string> {
try {
// Extract loio from the result ID
const loio = resultId.replace('sap-help-', '');
if (!loio || loio === resultId) {
throw new Error("Invalid SAP Help result ID. Use an ID from sap_help_search results.");
}
// First try to get from cache
const cache = global.sapHelpSearchCache || new Map();
let hit = cache.get(loio);
if (!hit) {
// If not in cache, search again to get the full hit data
const searchParams = {
transtype: "standard,html,pdf,others",
state: "PRODUCTION,TEST,DRAFT",
product: "",
version: "",
q: loio, // Search by loio to find the specific document
to: "19",
area: "content",
advancedSearch: "0",
excludeNotSearchable: "1",
language: "en-US",
};
const searchUrl = `${BASE}/http.svc/elasticsearch?${toQuery(searchParams)}`;
const searchResponse = await fetch(searchUrl, {
headers: {
Accept: "application/json",
"User-Agent": "mcp-sap-docs/help-get",
Referer: BASE,
},
});
if (!searchResponse.ok) {
throw new Error(`Failed to find document: ${searchResponse.status} ${searchResponse.statusText}`);
}
const searchData: SapHelpSearchResponse = await searchResponse.json();
const results = searchData?.data?.results || [];
hit = results.find(r => r.loio === loio);
if (!hit) {
throw new Error(`Document with loio ${loio} not found`);
}
}
// Prepare metadata request parameters
const topic_url = `${hit.loio}.html`;
let product_url = hit.productId;
let deliverable_url;
try {
const { productUrlSeg, deliverableLoio } = parseDocsPathParts(hit.url);
deliverable_url = deliverableLoio;
if (!product_url) product_url = productUrlSeg;
} catch (e) {
if (!product_url) {
throw new Error("Could not determine product_url from hit; missing productId and unparsable url");
}
}
const language = hit.language || "en-US";
// Get deliverable metadata
const metadataParams = {
product_url,
topic_url,
version: "LATEST",
loadlandingpageontopicnotfound: "true",
deliverable_url,
language,
deliverableInfo: "1",
toc: "1",
};
const metadataUrl = `${BASE}/http.svc/deliverableMetadata?${toQuery(metadataParams)}`;
const metadataResponse = await fetch(metadataUrl, {
headers: {
Accept: "application/json",
"User-Agent": "mcp-sap-docs/help-metadata",
Referer: BASE,
},
});
if (!metadataResponse.ok) {
throw new Error(`Metadata request failed: ${metadataResponse.status} ${metadataResponse.statusText}`);
}
const metadataData: SapHelpMetadataResponse = await metadataResponse.json();
const deliverable_id = metadataData?.data?.deliverable?.id;
const buildNo = metadataData?.data?.deliverable?.buildNo;
const file_path = metadataData?.data?.filePath || topic_url;
if (!deliverable_id || !buildNo || !file_path) {
throw new Error("Missing required metadata: deliverable_id, buildNo, or file_path");
}
// Get page content
const pageParams = {
deliverableInfo: "1",
deliverable_id,
buildNo,
file_path,
};
const pageUrl = `${BASE}/http.svc/pagecontent?${toQuery(pageParams)}`;
const pageResponse = await fetch(pageUrl, {
headers: {
Accept: "application/json",
"User-Agent": "mcp-sap-docs/help-content",
Referer: BASE,
},
});
if (!pageResponse.ok) {
throw new Error(`Page content request failed: ${pageResponse.status} ${pageResponse.statusText}`);
}
const pageData: SapHelpPageContentResponse = await pageResponse.json();
const title = pageData?.data?.currentPage?.t || pageData?.data?.deliverable?.title || hit.title;
const bodyHtml = pageData?.data?.body || "";
if (!bodyHtml) {
return `# ${title}\n\nNo content available for this page.`;
}
// Convert HTML to readable text while preserving structure
const cleanText = bodyHtml
.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '') // Remove scripts
.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '') // Remove styles
.replace(/<h([1-6])[^>]*>/gi, (_, level) => '\n' + '#'.repeat(parseInt(level)) + ' ') // Convert headings
.replace(/<\/h[1-6]>/gi, '\n') // Close headings
.replace(/<p[^>]*>/gi, '\n') // Paragraphs
.replace(/<\/p>/gi, '\n')
.replace(/<br[^>]*>/gi, '\n') // Line breaks
.replace(/<li[^>]*>/gi, '• ') // List items
.replace(/<\/li>/gi, '\n')
.replace(/<code[^>]*>/gi, '`') // Inline code
.replace(/<\/code>/gi, '`')
.replace(/<pre[^>]*>/gi, '\n```\n') // Code blocks
.replace(/<\/pre>/gi, '\n```\n')
.replace(/<[^>]+>/g, '') // Remove remaining HTML tags
.replace(/\s*\n\s*\n\s*/g, '\n\n') // Clean up multiple newlines
.replace(/^\s+|\s+$/g, '') // Trim
.trim();
// Build the full content with metadata
const fullContent = `# ${title}
**Source:** SAP Help Portal
**URL:** ${ensureAbsoluteUrl(hit.url)}
**Product:** ${hit.product || hit.productId || "Unknown"}
**Version:** ${hit.version || hit.versionId || "Latest"}
**Language:** ${hit.language || "en-US"}
${hit.snippet ? `**Summary:** ${hit.snippet}` : ''}
---
${cleanText}
---
*This content is from the SAP Help Portal and represents official SAP documentation.*`;
// Apply intelligent truncation if content is too large
const truncationResult = truncateContent(fullContent);
return truncationResult.content;
} catch (error: any) {
throw new Error(`Failed to get SAP Help content: ${error.message}`);
}
}
```
--------------------------------------------------------------------------------
/index.html:
--------------------------------------------------------------------------------
```html
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>SAP Documentation MCP Server</title>
<style>
body {
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 'Ubuntu', 'Cantarell', sans-serif;
line-height: 1.6;
color: #24292e;
max-width: 1200px;
margin: 0 auto;
padding: 20px;
background-color: #f6f8fa;
}
.container {
background-color: white;
padding: 40px;
border-radius: 8px;
box-shadow: 0 1px 3px rgba(0,0,0,0.12), 0 1px 2px rgba(0,0,0,0.24);
}
h1 {
color: #0366d6;
border-bottom: 2px solid #e1e4e8;
padding-bottom: 10px;
}
h2 {
color: #24292e;
margin-top: 30px;
border-bottom: 1px solid #e1e4e8;
padding-bottom: 5px;
}
.highlight {
background-color: #f1f8ff;
border: 1px solid #c8e1ff;
border-radius: 6px;
padding: 16px;
margin: 16px 0;
}
.server-url {
font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace;
background-color: #f6f8fa;
padding: 2px 4px;
border-radius: 3px;
font-size: 14px;
}
.feature-grid {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
gap: 20px;
margin: 20px 0;
}
.feature-card {
background-color: #f6f8fa;
padding: 20px;
border-radius: 6px;
border-left: 4px solid #0366d6;
}
.feature-card h3 {
margin-top: 0;
color: #0366d6;
}
code {
background-color: rgba(27,31,35,0.05);
padding: 2px 4px;
border-radius: 3px;
font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace;
font-size: 14px;
}
.status-links {
display: flex;
gap: 10px;
flex-wrap: wrap;
margin: 20px 0;
}
.status-link {
background-color: #0366d6;
color: white;
padding: 8px 16px;
text-decoration: none;
border-radius: 6px;
font-size: 14px;
transition: background-color 0.2s;
}
.status-link:hover {
background-color: #0256cc;
}
.coverage-stats {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
gap: 15px;
margin: 20px 0;
}
.stat-card {
text-align: center;
padding: 15px;
background-color: #f1f8ff;
border-radius: 6px;
}
.stat-number {
font-size: 24px;
font-weight: bold;
color: #0366d6;
display: block;
}
.tools-list {
list-style: none;
padding: 0;
display: grid;
grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
gap: 10px;
}
.tools-list li {
background-color: #f6f8fa;
padding: 10px 15px;
border-radius: 6px;
border-left: 3px solid #28a745;
}
.example-section {
background-color: #f8f9fa;
border-radius: 6px;
padding: 20px;
margin: 20px 0;
}
.example-section h3 {
margin-top: 0;
color: #24292e;
}
.example-queries {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
gap: 15px;
}
.query-card {
background-color: white;
padding: 15px;
border-radius: 6px;
border: 1px solid #e1e4e8;
}
.query-card strong {
color: #0366d6;
}
</style>
</head>
<body>
<div class="container">
<h1>🚀 SAP Documentation MCP Server</h1>
<div class="highlight">
<strong>Welcome!</strong> This server provides unified access to official SAP documentation, community content, and help portal resources through the Model Context Protocol (MCP).
</div>
<div class="status-links">
<a href="/status" class="status-link">📊 Server Status</a>
<a href="/healthz" class="status-link">🏥 Health Check</a>
<a href="https://mcp-sap-docs.marianzeis.de/mcp" class="status-link">🌐 Public MCP Endpoint</a>
</div>
<h2>🔗 Quick Start</h2>
<p>Connect your MCP client to this server using one of these methods:</p>
<div class="feature-grid">
<div class="feature-card">
<h3>🌐 Remote Connection (Recommended)</h3>
<p>Use the public MCP Streamable HTTP endpoint:</p>
<div class="server-url">https://mcp-sap-docs.marianzeis.de/mcp</div>
<p>Perfect for Claude Desktop, VS Code, Cursor, and other MCP clients.</p>
</div>
<div class="feature-card">
<h3>💻 Local STDIO</h3>
<p>Run locally with:</p>
<code>node dist/src/server.js</code>
<p>For development and local testing.</p>
</div>
<div class="feature-card">
<h3>🔄 Streamable HTTP</h3>
<p>Latest MCP protocol support:</p>
<code>http://127.0.0.1:3122/mcp</code>
<p>Enhanced session management and resumability.</p>
</div>
</div>
<h2>🛠️ Available Tools</h2>
<ul class="tools-list">
<li><strong>sap_docs_search</strong> - Unified search across SAPUI5/CAP/OpenUI5 APIs & samples, wdi5, and more</li>
<li><strong>sap_community_search</strong> - Real-time SAP Community posts with full content of top 3 results</li>
<li><strong>sap_help_search</strong> - Comprehensive search across all SAP Help Portal documentation</li>
<li><strong>sap_docs_get</strong> - Fetches full documents/snippets with smart formatting</li>
<li><strong>sap_help_get</strong> - Retrieves complete SAP Help pages with metadata</li>
</ul>
<h2>📚 Documentation Coverage</h2>
<div class="coverage-stats">
<div class="stat-card">
<span class="stat-number">1,485+</span>
SAPUI5 Documentation Files
</div>
<div class="stat-card">
<span class="stat-number">195+</span>
CAP Documentation Files
</div>
<div class="stat-card">
<span class="stat-number">500+</span>
OpenUI5 API Controls
</div>
<div class="stat-card">
<span class="stat-number">2,000+</span>
Sample Code Files
</div>
<div class="stat-card">
<span class="stat-number">Real-time</span>
Community Content
</div>
<div class="stat-card">
<span class="stat-number">Complete</span>
SAP Help Portal
</div>
</div>
<div class="example-section">
<h3>💡 Example Queries</h3>
<div class="example-queries">
<div class="query-card">
<strong>Official Documentation:</strong><br>
"How do I implement authentication in SAPUI5?"<br>
"Show me wdi5 testing examples for forms"<br>
"Find OpenUI5 button control examples"
</div>
<div class="query-card">
<strong>Community Knowledge:</strong><br>
"Latest CAP authentication best practices from community"<br>
"Community examples of OData batch operations"<br>
"Temporal data handling in CAP solutions"
</div>
<div class="query-card">
<strong>SAP Help Portal:</strong><br>
"How to configure S/4HANA Fiori Launchpad?"<br>
"BTP integration documentation for Analytics Cloud"<br>
"ABAP development best practices in S/4HANA"
</div>
</div>
</div>
<h2>🏛️ Architecture</h2>
<div class="feature-grid">
<div class="feature-card">
<h3>🧠 MCP Server</h3>
<p>Node.js/TypeScript server exposing SAP documentation resources and tools</p>
</div>
<div class="feature-card">
<h3>🔄 Streamable HTTP</h3>
<p>Latest MCP spec (2025-06-18) with session management and resumability</p>
</div>
<div class="feature-card">
<h3>🔍 Search Engine</h3>
<p>SQLite FTS5 + JSON indices for fast local search</p>
</div>
<div class="feature-card">
<h3>👥 Community Integration</h3>
<p>HTML scraping + LiQL API for full content retrieval</p>
</div>
<div class="feature-card">
<h3>📖 SAP Help Integration</h3>
<p>Private API access to help.sap.com content</p>
</div>
</div>
<h2>🔧 For Developers</h2>
<div class="highlight">
<h3>Build Commands:</h3>
<code>npm run build</code> - Compile TypeScript<br>
<code>npm run build:index</code> - Build search index<br>
<code>npm run build:fts</code> - Build FTS5 database<br><br>
<h3>Server Commands:</h3>
<code>npm start</code> - Start STDIO MCP server<br>
<code>npm run start:http</code> - Start HTTP status server (port 3001)<br>
<code>npm run start:streamable</code> - Start Streamable HTTP MCP server (port 3122)<br>
</div>
<div style="margin-top: 40px; padding-top: 20px; border-top: 1px solid #e1e4e8; text-align: center; color: #6a737d;">
<p>📊 <strong>Total Coverage:</strong> 4,180+ documentation files + real-time community & help portal content</p>
<p>🔗 <a href="https://github.com/marianfoo/mcp-sap-docs" style="color: #0366d6;">View on GitHub</a> |
📄 <a href="https://github.com/marianfoo/mcp-sap-docs/blob/main/README.md" style="color: #0366d6;">Full Documentation</a></p>
</div>
</div>
</body>
</html>
```
--------------------------------------------------------------------------------
/test/tools/run-tests.js:
--------------------------------------------------------------------------------
```javascript
// Unified test runner for MCP SAP Docs - supports both all tests and specific files
import { readdirSync } from 'node:fs';
import { join, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
import { startServerHttp, waitForStatus, stopServer, docsSearch } from '../_utils/httpClient.js';
// ANSI color codes
const colors = {
reset: '\x1b[0m',
bright: '\x1b[1m',
dim: '\x1b[2m',
red: '\x1b[31m',
green: '\x1b[32m',
yellow: '\x1b[33m',
blue: '\x1b[34m',
magenta: '\x1b[35m',
cyan: '\x1b[36m',
white: '\x1b[37m'
};
function colorize(text, color) {
return `${colors[color]}${text}${colors.reset}`;
}
const __filename = fileURLToPath(import.meta.url);
const ROOT = dirname(__filename);
const TOOLS_DIR = join(ROOT);
function listJsFiles(dir) {
const entries = readdirSync(dir, { withFileTypes: true });
const files = [];
for (const e of entries) {
const p = join(dir, e.name);
if (e.isDirectory()) files.push(...listJsFiles(p));
else if (e.isFile() && e.name.endsWith('.js')) files.push(p);
}
return files;
}
function parseArgs() {
const args = process.argv.slice(2);
const config = {
specificFile: null,
showHelp: false
};
for (let i = 0; i < args.length; i++) {
const arg = args[i];
if (arg === '--spec' && i + 1 < args.length) {
config.specificFile = args[i + 1];
i++; // Skip next argument since it's the file path
} else if (arg === '--help' || arg === '-h') {
config.showHelp = true;
}
}
return config;
}
function showHelp() {
console.log(colorize('MCP SAP Docs Test Runner', 'cyan'));
console.log('');
console.log(colorize('Usage:', 'yellow'));
console.log(' npm run test Run all test files');
console.log(' npm run test -- --spec <file-path> Run specific test file');
console.log('');
console.log(colorize('Examples:', 'yellow'));
console.log(' npm run test');
console.log(' npm run test:fast Skip build step');
console.log(' npm run test -- --spec search-cap-docs.js');
console.log(' npm run test:smoke Quick smoke test');
console.log('');
console.log(colorize('Available test files:', 'yellow'));
const allFiles = listJsFiles(TOOLS_DIR)
.filter(p => !p.endsWith('run-all.js') && !p.endsWith('run-single.js') && !p.endsWith('run-tests.js'));
allFiles.forEach(f => {
// Show relative path from project root
const relativePath = f.replace(process.cwd() + '/', '');
console.log(colorize(` ${relativePath}`, 'cyan'));
});
}
function findTestFile(pattern) {
const allFiles = listJsFiles(TOOLS_DIR)
.filter(p => !p.endsWith('run-all.js') && !p.endsWith('run-single.js') && !p.endsWith('run-tests.js'));
// Try different matching strategies
let matches = [];
// 1. Exact path match (relative to project root or absolute)
if (pattern.startsWith('/')) {
matches = allFiles.filter(f => f === pattern);
} else {
// Try as relative path from project root
const fullPattern = join(process.cwd(), pattern);
matches = allFiles.filter(f => f === fullPattern);
}
// 2. If no exact match, try partial path matching
if (matches.length === 0) {
matches = allFiles.filter(f => f.includes(pattern));
}
// 3. If still no match, try just filename matching
if (matches.length === 0) {
matches = allFiles.filter(f => f.split('/').pop() === pattern);
}
if (matches.length === 0) {
console.log(colorize(`❌ No test file found matching: ${pattern}`, 'red'));
console.log(colorize('Available test files:', 'yellow'));
allFiles.forEach(f => {
const relativePath = f.replace(process.cwd() + '/', '');
console.log(colorize(` ${relativePath}`, 'cyan'));
});
process.exit(1);
}
if (matches.length > 1) {
console.log(colorize(`⚠️ Multiple files match "${pattern}":`, 'yellow'));
matches.forEach(f => {
const relativePath = f.replace(process.cwd() + '/', '');
console.log(colorize(` ${relativePath}`, 'cyan'));
});
console.log(colorize('Please be more specific.', 'yellow'));
process.exit(1);
}
return matches[0];
}
async function runTestFile(filePath, fileName) {
console.log(colorize(`📁 Running ${fileName}`, 'blue'));
console.log(colorize('─'.repeat(50), 'dim'));
// Load and run the test file
const mod = await import(fileURLToPath(new URL(filePath, import.meta.url)));
const cases = (mod.default || []).flat();
if (cases.length === 0) {
console.log(colorize('⚠️ No test cases found in file', 'yellow'));
return { tests: 0, failures: 0 };
}
let fileFailures = 0;
let fileTests = 0;
for (const c of cases) {
try {
if (typeof c.validate === 'function') {
// New path: custom validator gets helpers, uses existing server
const res = await c.validate({ docsSearch });
if (res && typeof res === 'object' && res.skipped) {
const reason = res.message ? ` - ${res.message}` : '';
console.log(` ${colorize('⚠️', 'yellow')} ${colorize(c.name, 'white')} (skipped${reason})`);
continue;
}
fileTests++; // Only count tests that are actually executed
const passed = typeof res === 'object' ? !!res.passed : !!res;
if (!passed) {
const msg = (res && res.message) ? ` - ${res.message}` : '';
throw new Error(`custom validator failed${msg}`);
}
console.log(` ${colorize('✅', 'green')} ${colorize(c.name, 'white')}`);
} else {
// Legacy path: expectIncludes (kept for existing tests)
const text = await docsSearch(c.query);
if (c.skipIfNoResults && /No results found for/.test(text)) {
console.log(` ${colorize('⚠️', 'yellow')} ${colorize(c.name, 'white')} (skipped - no results available)`);
continue;
}
fileTests++; // Only count tests that are actually executed
// Check expectIncludes
if (c.expectIncludes) {
const checks = Array.isArray(c.expectIncludes) ? c.expectIncludes : [c.expectIncludes];
const ok = checks.every(expectedFragment => {
// Direct match (exact inclusion)
if (text.includes(expectedFragment)) {
return true;
}
// If expected fragment is a parent document (no #), check if any section from that document is found
if (!expectedFragment.includes('#')) {
// Look for any section that starts with the expected parent document path followed by #
const sectionPattern = new RegExp(expectedFragment.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '#[^\\s]*', 'g');
return sectionPattern.test(text);
}
return false;
});
if (!ok) throw new Error(`expected fragment(s) not found: ${checks.join(', ')}`);
}
// Check expectContains (for URL verification)
if (c.expectContains) {
const containsChecks = Array.isArray(c.expectContains) ? c.expectContains : [c.expectContains];
const containsOk = containsChecks.every(expectedContent => {
return text.includes(expectedContent);
});
if (!containsOk) throw new Error(`expected content not found: ${containsChecks.join(', ')}`);
}
// Check expectUrlPattern (for URL format verification)
if (c.expectUrlPattern) {
// Extract URLs from the response using the 🔗 emoji
const urlRegex = /🔗\s+(https?:\/\/[^\s\n]+)/g;
const urls = [];
let match;
while ((match = urlRegex.exec(text)) !== null) {
urls.push(match[1]);
}
if (urls.length === 0) {
throw new Error('no URLs found in response (expected URL pattern)');
}
const urlPattern = c.expectUrlPattern;
const matchingUrl = urls.some(url => {
if (typeof urlPattern === 'string') {
return url.includes(urlPattern) || new RegExp(urlPattern).test(url);
}
return urlPattern.test(url);
});
if (!matchingUrl) {
throw new Error(`no URL matching pattern "${urlPattern}" found. URLs found: ${urls.join(', ')}`);
}
}
// Check expectPattern (for general regex pattern matching)
if (c.expectPattern) {
const pattern = c.expectPattern;
if (!pattern.test(text)) {
throw new Error(`text does not match expected pattern: ${pattern}`);
}
}
console.log(` ${colorize('✅', 'green')} ${colorize(c.name, 'white')}`);
}
} catch (err) {
fileFailures++;
console.log(` ${colorize('❌', 'red')} ${colorize(c.name, 'white')}: ${colorize(err?.message || err, 'red')}`);
}
}
return { tests: fileTests, failures: fileFailures };
}
async function runTests() {
const config = parseArgs();
if (config.showHelp) {
showHelp();
process.exit(0);
}
let testFiles = [];
if (config.specificFile) {
// Run specific test file
const testFile = findTestFile(config.specificFile);
const fileName = testFile.split('/').pop();
console.log(colorize(`🚀 Running specific test: ${fileName}`, 'cyan'));
testFiles = [testFile];
} else {
// Run all test files
console.log(colorize('🚀 Starting MCP SAP Docs test suite...', 'cyan'));
testFiles = listJsFiles(TOOLS_DIR)
.filter(p => {
const fileName = p.split('/').pop();
// Skip runner scripts and utility files
return !fileName.startsWith('run-') &&
!fileName.includes('test-with-reranker') &&
fileName.endsWith('.js');
})
.sort();
}
// Start HTTP server
const server = startServerHttp();
let totalFailures = 0;
let totalTests = 0;
try {
console.log(colorize('⏳ Waiting for server to be ready...', 'yellow'));
await waitForStatus();
console.log(colorize('✅ Server ready!\n', 'green'));
for (const file of testFiles) {
const fileName = file.split('/').pop();
// Add spacing between files when running multiple
if (testFiles.length > 1) {
console.log('');
}
const result = await runTestFile(file, fileName);
totalTests += result.tests;
totalFailures += result.failures;
}
} finally {
await stopServer(server);
}
console.log(colorize('\n' + '═'.repeat(60), 'dim'));
if (totalFailures) {
console.log(`${colorize('❌ Test Results:', 'red')} ${colorize(`${totalFailures}/${totalTests} tests failed`, 'red')}`);
process.exit(1);
} else {
console.log(`${colorize('🎉 Test Results:', 'green')} ${colorize(`All ${totalTests} tests passed!`, 'green')}`);
}
}
runTests().catch(err => {
console.error(colorize('Fatal error:', 'red'), err);
process.exit(1);
});
```
--------------------------------------------------------------------------------
/test/validate-urls.ts:
--------------------------------------------------------------------------------
```typescript
#!/usr/bin/env node
/**
* URL Validation Script
* Tests random URLs from each documentation source to verify they're not 404
*/
import { fileURLToPath } from 'url';
import { dirname, join } from 'path';
import fs from 'fs/promises';
import { existsSync } from 'fs';
import { generateDocumentationUrl } from '../src/lib/url-generation/index.js';
import { getDocUrlConfig, getSourcePath } from '../src/lib/metadata.js';
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
const PROJECT_ROOT = join(__dirname, '..');
const DATA_DIR = join(PROJECT_ROOT, 'dist', 'data');
interface TestResult {
source: string;
url: string;
status: number;
ok: boolean;
error?: string;
docTitle: string;
relFile: string;
responseTime: number;
}
interface LibraryBundle {
id: string;
name: string;
description: string;
docs: {
id: string;
title: string;
description: string;
snippetCount: number;
relFile: string;
}[];
}
// Colors for console output
const colors = {
reset: '\x1b[0m',
red: '\x1b[31m',
green: '\x1b[32m',
yellow: '\x1b[33m',
blue: '\x1b[34m',
magenta: '\x1b[35m',
cyan: '\x1b[36m',
bold: '\x1b[1m',
dim: '\x1b[2m'
};
async function loadIndex(): Promise<Record<string, LibraryBundle>> {
const indexPath = join(DATA_DIR, 'index.json');
if (!existsSync(indexPath)) {
throw new Error(`Index file not found: ${indexPath}. Run 'npm run build' first.`);
}
const raw = await fs.readFile(indexPath, 'utf8');
return JSON.parse(raw) as Record<string, LibraryBundle>;
}
function getRandomItems<T>(array: T[], count: number): T[] {
const shuffled = [...array].sort(() => 0.5 - Math.random());
return shuffled.slice(0, Math.min(count, array.length));
}
async function getDocumentContent(libraryId: string, relFile: string): Promise<string> {
const sourcePath = getSourcePath(libraryId);
if (!sourcePath) {
throw new Error(`Unknown library ID: ${libraryId}`);
}
const fullPath = join(PROJECT_ROOT, 'sources', sourcePath, relFile);
if (!existsSync(fullPath)) {
return '# No content available';
}
return await fs.readFile(fullPath, 'utf8');
}
async function testUrl(url: string, timeout: number = 10000): Promise<{ status: number; ok: boolean; error?: string; responseTime: number }> {
const startTime = Date.now();
try {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), timeout);
const response = await fetch(url, {
method: 'HEAD', // Use HEAD to avoid downloading full content
signal: controller.signal,
headers: {
'User-Agent': 'SAP-Docs-MCP-URL-Validator/1.0'
}
});
clearTimeout(timeoutId);
const responseTime = Date.now() - startTime;
return {
status: response.status,
ok: response.ok,
responseTime
};
} catch (error: any) {
const responseTime = Date.now() - startTime;
if (error.name === 'AbortError') {
return {
status: 0,
ok: false,
error: 'Timeout',
responseTime
};
}
return {
status: 0,
ok: false,
error: error.message || 'Network error',
responseTime
};
}
}
async function generateUrlForDoc(libraryId: string, doc: any): Promise<string | null> {
const config = getDocUrlConfig(libraryId);
if (!config) {
console.warn(`${colors.yellow}⚠️ No URL config for ${libraryId}${colors.reset}`);
return null;
}
try {
const content = await getDocumentContent(libraryId, doc.relFile);
return generateDocumentationUrl(libraryId, doc.relFile, content, config);
} catch (error) {
console.warn(`${colors.yellow}⚠️ Could not read content for ${doc.relFile}: ${error}${colors.reset}`);
return null;
}
}
async function validateSourceUrls(library: LibraryBundle, sampleSize: number = 5): Promise<TestResult[]> {
console.log(`\n${colors.cyan}📚 Testing ${library.name} (${library.id})${colors.reset}`);
console.log(`${colors.dim} ${library.description}${colors.reset}`);
// Get random sample of documents
const randomDocs = getRandomItems(library.docs, sampleSize);
console.log(`${colors.blue} Selected ${randomDocs.length} random documents${colors.reset}`);
const results: TestResult[] = [];
const promises = randomDocs.map(async (doc) => {
const url = await generateUrlForDoc(library.id, doc);
if (!url) {
return {
source: library.id,
url: 'N/A',
status: 0,
ok: false,
error: 'Could not generate URL',
docTitle: doc.title,
relFile: doc.relFile,
responseTime: 0
};
}
console.log(`${colors.dim} Testing: ${url}${colors.reset}`);
const testResult = await testUrl(url);
return {
source: library.id,
url,
status: testResult.status,
ok: testResult.ok,
error: testResult.error,
docTitle: doc.title,
relFile: doc.relFile,
responseTime: testResult.responseTime
};
});
// Wait for all tests to complete
const testResults = await Promise.all(promises);
results.push(...testResults);
// Display results for this source
const successful = results.filter(r => r.ok).length;
const failed = results.filter(r => !r.ok).length;
console.log(`${colors.bold} Results: ${colors.green}✅ ${successful} OK${colors.reset}${colors.bold}, ${colors.red}❌ ${failed} Failed${colors.reset}`);
// Show detailed results
results.forEach(result => {
const statusColor = result.ok ? colors.green : colors.red;
const statusIcon = result.ok ? '✅' : '❌';
const statusText = result.status > 0 ? result.status.toString() : (result.error || 'ERROR');
console.log(` ${statusIcon} ${statusColor}[${statusText}]${colors.reset} ${result.docTitle}`);
console.log(` ${colors.dim}${result.url}${colors.reset}`);
if (!result.ok && result.error) {
console.log(` ${colors.red}Error: ${result.error}${colors.reset}`);
}
if (result.responseTime > 0) {
console.log(` ${colors.dim}Response time: ${result.responseTime}ms${colors.reset}`);
}
});
return results;
}
async function generateSummaryReport(allResults: TestResult[]) {
console.log(`\n${colors.bold}${colors.cyan}📊 SUMMARY REPORT${colors.reset}`);
console.log(`${'='.repeat(60)}`);
const totalTests = allResults.length;
const successfulTests = allResults.filter(r => r.ok).length;
const failedTests = allResults.filter(r => !r.ok).length;
const successRate = totalTests > 0 ? ((successfulTests / totalTests) * 100).toFixed(1) : '0.0';
console.log(`${colors.bold}Overall Results:${colors.reset}`);
console.log(` Total URLs tested: ${colors.bold}${totalTests}${colors.reset}`);
console.log(` Successful: ${colors.green}${successfulTests}${colors.reset}`);
console.log(` Failed: ${colors.red}${failedTests}${colors.reset}`);
console.log(` Success rate: ${colors.bold}${successRate}%${colors.reset}`);
// Group by source
const bySource = allResults.reduce((acc, result) => {
if (!acc[result.source]) {
acc[result.source] = { total: 0, successful: 0, failed: 0 };
}
acc[result.source].total++;
if (result.ok) {
acc[result.source].successful++;
} else {
acc[result.source].failed++;
}
return acc;
}, {} as Record<string, { total: number; successful: number; failed: number }>);
console.log(`\n${colors.bold}By Source:${colors.reset}`);
Object.entries(bySource).forEach(([source, stats]) => {
const rate = ((stats.successful / stats.total) * 100).toFixed(1);
const rateColor = stats.successful === stats.total ? colors.green :
stats.successful > stats.total / 2 ? colors.yellow : colors.red;
console.log(` ${source}: ${rateColor}${rate}%${colors.reset} (${colors.green}${stats.successful}${colors.reset}/${stats.total})`);
});
// Show failed URLs
const failed = allResults.filter(r => !r.ok);
if (failed.length > 0) {
console.log(`\n${colors.bold}${colors.red}❌ Failed URLs:${colors.reset}`);
failed.forEach(result => {
console.log(` ${colors.red}[${result.status || 'ERROR'}]${colors.reset} ${result.url}`);
console.log(` ${colors.dim}${result.docTitle} (${result.source})${colors.reset}`);
if (result.error) {
console.log(` ${colors.red}${result.error}${colors.reset}`);
}
});
}
// Performance stats
const responseTimes = allResults.filter(r => r.responseTime > 0).map(r => r.responseTime);
if (responseTimes.length > 0) {
const avgResponseTime = Math.round(responseTimes.reduce((a, b) => a + b, 0) / responseTimes.length);
const maxResponseTime = Math.max(...responseTimes);
console.log(`\n${colors.bold}Performance:${colors.reset}`);
console.log(` Average response time: ${avgResponseTime}ms`);
console.log(` Slowest response: ${maxResponseTime}ms`);
}
}
async function main() {
console.log(`${colors.bold}${colors.blue}🔗 SAP Docs MCP - URL Validation Tool${colors.reset}`);
console.log(`Testing random URLs from each documentation source...\n`);
try {
const index = await loadIndex();
const sources = Object.values(index);
console.log(`${colors.bold}Found ${sources.length} documentation sources:${colors.reset}`);
sources.forEach(lib => {
const hasUrlConfig = getDocUrlConfig(lib.id) !== null;
const configStatus = hasUrlConfig ? `${colors.green}✅${colors.reset}` : `${colors.red}❌${colors.reset}`;
console.log(` ${configStatus} ${lib.name} (${lib.id}) - ${lib.docs.length} docs`);
});
const sourcesWithUrls = sources.filter(lib => getDocUrlConfig(lib.id) !== null);
if (sourcesWithUrls.length === 0) {
console.log(`${colors.red}❌ No sources have URL configuration. Cannot test URLs.${colors.reset}`);
process.exit(1);
}
console.log(`\n${colors.bold}Testing URLs for ${sourcesWithUrls.length} sources with URL configuration...${colors.reset}`);
// Test each source
const allResults: TestResult[] = [];
for (const library of sourcesWithUrls) {
try {
const results = await validateSourceUrls(library, 5);
allResults.push(...results);
} catch (error) {
console.error(`${colors.red}❌ Error testing ${library.name}: ${error}${colors.reset}`);
}
}
// Generate summary report
await generateSummaryReport(allResults);
// Exit with appropriate code
const hasFailures = allResults.some(r => !r.ok);
if (hasFailures) {
console.log(`\n${colors.yellow}⚠️ Some URLs failed validation. Check the results above.${colors.reset}`);
process.exit(1);
} else {
console.log(`\n${colors.green}🎉 All URLs validated successfully!${colors.reset}`);
process.exit(0);
}
} catch (error) {
console.error(`${colors.red}❌ Error: ${error}${colors.reset}`);
process.exit(1);
}
}
// Handle CLI usage
if (process.argv[1] === fileURLToPath(import.meta.url)) {
main().catch(console.error);
}
```
--------------------------------------------------------------------------------
/.github/workflows/deploy-mcp-sap-docs.yml:
--------------------------------------------------------------------------------
```yaml
name: Deploy MCP stack
on:
push:
branches: [ main ]
# Allow manual triggering
workflow_dispatch:
jobs:
deploy:
runs-on: ubuntu-22.04
# 👇 must match the environment where you stored the secrets
environment:
name: remove server
steps:
- name: Check out repo (for action context only)
uses: actions/checkout@v4
with:
# Fetch more than just the triggering commit to handle workflow reruns
fetch-depth: 10
- name: Ensure we have the latest main branch
run: |
git fetch origin main
git checkout main
git reset --hard origin/main
echo "Current HEAD: $(git rev-parse HEAD)"
echo "Latest main: $(git rev-parse origin/main)"
- name: Preflight verify required secrets are present
run: |
set -euo pipefail
for s in SERVER_IP SERVER_USERNAME SSH_PRIVATE_KEY; do
[ -n "${!s}" ] || { echo "Missing $s"; exit 1; }
done
env:
SERVER_IP: ${{ secrets.SERVER_IP }}
SERVER_USERNAME: ${{ secrets.SERVER_USERNAME }}
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
- name: Auto-increment version
run: |
# Get current version and increment patch
CURRENT_VERSION=$(node -p "require('./package.json').version")
echo "Current version: $CURRENT_VERSION"
# Extract major.minor.patch
IFS='.' read -ra PARTS <<< "$CURRENT_VERSION"
MAJOR=${PARTS[0]}
MINOR=${PARTS[1]}
PATCH=${PARTS[2]}
# Increment patch version
NEW_PATCH=$((PATCH + 1))
NEW_VERSION="$MAJOR.$MINOR.$NEW_PATCH"
echo "New version: $NEW_VERSION"
echo "NEW_VERSION=$NEW_VERSION" >> $GITHUB_ENV
# Update package.json
npm version $NEW_VERSION --no-git-tag-version
- name: Update hardcoded version in streamable server
run: |
# Update the hardcoded version in streamable-http-server.ts
sed -i 's/const VERSION = "[0-9]*\.[0-9]*\.[0-9]*";/const VERSION = "'${{ env.NEW_VERSION }}'";/g' src/streamable-http-server.ts
- name: Commit version bump
id: version_bump
continue-on-error: true
run: |
set -e
git config --local user.email "[email protected]"
git config --local user.name "GitHub Action"
# Check if there are any changes to commit
git add package.json package-lock.json src/streamable-http-server.ts
if ! git diff --staged --quiet; then
if git commit -m "chore: bump version to ${{ env.NEW_VERSION }} [skip ci]"; then
echo "Version bump committed successfully"
# Handle potential conflicts from concurrent pushes or workflow reruns
echo "Attempting to push version bump..."
if ! git push; then
echo "Push failed, likely due to remote changes. Pulling and retrying..."
git fetch origin main
# Try rebase first
if git rebase origin/main; then
echo "Rebase successful, pushing again..."
git push
else
echo "Rebase failed, attempting merge strategy..."
git rebase --abort
git merge origin/main --no-edit
git push
fi
fi
echo "✅ Version bump pushed successfully"
else
echo "ℹ️ No version changes to commit"
fi
else
echo "ℹ️ No version changes detected"
fi
- name: Handle version bump failure
if: steps.version_bump.outcome == 'failure'
run: |
echo "⚠️ Version bump failed, but continuing with deployment"
echo "This can happen with concurrent workflows or when rerunning failed deployments"
echo "The deployment will proceed with the current version"
# Reset any partial git state
git reset --hard HEAD
git clean -fd
- name: Deploy to server via SSH
uses: appleboy/[email protected]
env:
NEW_VERSION: ${{ env.NEW_VERSION }}
with:
host: ${{ secrets.SERVER_IP }}
username: ${{ secrets.SERVER_USERNAME }}
key: ${{ secrets.SSH_PRIVATE_KEY }}
envs: NEW_VERSION
script: |
set -Eeuo pipefail
echo "==> Database Health Pre-Check"
cd /opt/mcp-sap/mcp-sap-docs || { echo "Directory not found, will be created"; }
# Function to check SQLite database integrity
check_db_integrity() {
local db_path="$1"
if [ -f "$db_path" ]; then
echo "🔍 Checking database integrity: $db_path"
if sqlite3 "$db_path" "PRAGMA integrity_check;" 2>/dev/null | grep -q "ok"; then
echo "✅ Database integrity OK"
return 0
else
echo "❌ Database corruption detected"
return 1
fi
else
echo "ℹ️ Database file does not exist: $db_path"
return 1
fi
}
# Check existing database and create backup
DB_PATH="/opt/mcp-sap/mcp-sap-docs/dist/data/docs.sqlite"
if [ -f "$DB_PATH" ]; then
if ! check_db_integrity "$DB_PATH"; then
echo "==> Database corruption detected - will rebuild"
rm -f "$DB_PATH"
else
echo "==> Creating database backup before deployment"
BACKUP_PATH="/opt/mcp-sap/backups/deploy-backup-$(date +%Y%m%d-%H%M%S).sqlite"
mkdir -p /opt/mcp-sap/backups
cp "$DB_PATH" "$BACKUP_PATH"
echo "✅ Database backed up to $BACKUP_PATH"
# Keep only last 5 backups
ls -t /opt/mcp-sap/backups/deploy-backup-*.sqlite 2>/dev/null | tail -n +6 | xargs -r rm --
fi
fi
echo "==> Ensure base path exists and owned by user"
sudo mkdir -p /opt/mcp-sap
sudo chown -R "$USER":"$USER" /opt/mcp-sap
echo "==> Clone or update repo (with submodules)"
if [ -d /opt/mcp-sap/mcp-sap-docs/.git ]; then
cd /opt/mcp-sap/mcp-sap-docs
git config --global url."https://github.com/".insteadOf [email protected]:
git fetch --prune
git reset --hard origin/main
else
cd /opt/mcp-sap
git config --global url."https://github.com/".insteadOf [email protected]:
git clone https://github.com/marianfoo/mcp-sap-docs.git
cd mcp-sap-docs
fi
echo "==> Deploying version: $NEW_VERSION"
echo "==> Configure BM25 search environment"
# Ensure metadata file exists for centralized configuration
[ -f /opt/mcp-sap/mcp-sap-docs/data/metadata.json ] || echo "Metadata file will be created during build"
echo "==> Check system resources before build"
AVAILABLE_MB=$(df /opt/mcp-sap --output=avail -m | tail -n1)
if [ "$AVAILABLE_MB" -lt 1000 ]; then
echo "❌ ERROR: Insufficient disk space. Available: ${AVAILABLE_MB}MB, Required: 1000MB"
exit 1
fi
echo "✅ Disk space OK: ${AVAILABLE_MB}MB available"
AVAILABLE_KB=$(awk '/MemAvailable/ { print $2 }' /proc/meminfo)
AVAILABLE_MB_MEM=$((AVAILABLE_KB / 1024))
if [ "$AVAILABLE_MB_MEM" -lt 512 ]; then
echo "❌ ERROR: Insufficient memory. Available: ${AVAILABLE_MB_MEM}MB, Required: 512MB"
exit 1
fi
echo "✅ Memory OK: ${AVAILABLE_MB_MEM}MB available"
echo "==> Stop MCP services gracefully before build"
pm2 stop mcp-sap-proxy mcp-sap-http mcp-sap-streamable || true
sleep 3
echo "==> Run setup (shallow, single-branch submodules + build)"
SKIP_NESTED_SUBMODULES=1 bash setup.sh
echo "==> Verify database integrity after build"
if ! check_db_integrity "$DB_PATH"; then
echo "❌ ERROR: Database corruption after build - deployment failed"
exit 1
fi
echo "✅ Database integrity verified after build"
echo "==> Create logs directory with proper permissions"
mkdir -p /opt/mcp-sap/logs
chown -R "$USER":"$USER" /opt/mcp-sap/logs
echo "==> (Re)start MCP services with BM25 search support"
# Proxy (SSE) on 127.0.0.1:18080; HTTP status on 127.0.0.1:3001; Streamable HTTP on 127.0.0.1:3122
pm2 start /opt/mcp-sap/mcp-sap-docs/ecosystem.config.cjs --only mcp-sap-proxy || pm2 restart mcp-sap-proxy
pm2 start /opt/mcp-sap/mcp-sap-docs/ecosystem.config.cjs --only mcp-sap-http || pm2 restart mcp-sap-http
pm2 start /opt/mcp-sap/mcp-sap-docs/ecosystem.config.cjs --only mcp-sap-streamable || pm2 restart mcp-sap-streamable
pm2 save
echo "==> Enhanced health checks with database verification"
sleep 5
for i in $(seq 1 30); do curl -fsS http://127.0.0.1:18080/status >/dev/null && break || sleep 2; done
curl -fsS http://127.0.0.1:18080/status
for i in $(seq 1 30); do curl -fsS http://127.0.0.1:3001/status >/dev/null && break || sleep 2; done
curl -fsS http://127.0.0.1:3001/status
# Streamable HTTP server health on 127.0.0.1:3122
for i in $(seq 1 30); do curl -fsS http://127.0.0.1:3122/health >/dev/null && break || sleep 2; done
curl -fsS http://127.0.0.1:3122/health
# Test actual search functionality to ensure no SQLite corruption
echo "==> Testing search functionality"
SEARCH_TEST=$(curl -s -X POST http://127.0.0.1:3001/mcp -H "Content-Type: application/json" -d '{"role": "user", "content": "test search"}')
if echo "$SEARCH_TEST" | grep -q "SqliteError\|SQLITE_CORRUPT\|Tool execution failed"; then
echo "❌ ERROR: Search test failed - possible database corruption"
echo "Response: $SEARCH_TEST"
exit 1
fi
echo "✅ Search functionality verified - no corruption detected"
echo "==> Final database integrity check"
if ! check_db_integrity "$DB_PATH"; then
echo "❌ WARNING: Database corruption detected after deployment"
# Don't fail deployment, but alert
else
echo "✅ Final database integrity check passed"
fi
echo "✅ Deployment completed successfully - Version: $NEW_VERSION"
```
--------------------------------------------------------------------------------
/src/http-server.ts:
--------------------------------------------------------------------------------
```typescript
import { createServer } from "http";
import { readFileSync, statSync, existsSync, readdirSync } from "fs";
import { fileURLToPath } from "url";
import { dirname, join, resolve } from "path";
import { execSync } from "child_process";
import { searchLibraries } from "./lib/localDocs.js";
import { search } from "./lib/search.js";
import { CONFIG } from "./lib/config.js";
import { loadMetadata, getDocUrlConfig } from "./lib/metadata.js";
import { generateDocumentationUrl, formatSearchResult } from "./lib/url-generation/index.js";
const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);
// ---- build/package meta -----------------------------------------------------
let packageInfo: { version: string; name: string } = { version: "unknown", name: "mcp-sap-docs" };
try {
const packagePath = join(__dirname, "../../package.json");
packageInfo = JSON.parse(readFileSync(packagePath, "utf8"));
} catch (error) {
console.warn("Could not read package.json:", error instanceof Error ? error.message : "Unknown error");
}
const buildTimestamp = new Date().toISOString();
// ---- helpers ----------------------------------------------------------------
function safeExec(cmd: string, cwd?: string) {
try {
return execSync(cmd, { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], cwd }).trim();
} catch {
return "";
}
}
// Handle both normal repos and submodules where `.git` is a FILE with `gitdir: …`
function resolveGitDir(repoPath: string): string | null {
const dotGit = join(repoPath, ".git");
if (!existsSync(dotGit)) return null;
const st = statSync(dotGit);
if (st.isDirectory()) return dotGit;
// .git is a file that points to the real gitdir
const content = readFileSync(dotGit, "utf8");
const m = content.match(/^gitdir:\s*(.+)$/m);
if (!m) return null;
return resolve(repoPath, m[1]);
}
function readGitMeta(repoPath: string) {
try {
const gitDir = resolveGitDir(repoPath);
if (!gitDir) return { error: "No git dir" };
const headPath = join(gitDir, "HEAD");
const head = readFileSync(headPath, "utf8").trim();
if (head.startsWith("ref: ")) {
const ref = head.slice(5).trim();
const refPath = join(gitDir, ref);
const commit = readFileSync(refPath, "utf8").trim();
const date = safeExec(`git log -1 --format="%ci"`, repoPath);
return {
branch: ref.split("/").pop(),
commit: commit.substring(0, 7),
fullCommit: commit,
lastModified: date ? new Date(date).toISOString() : statSync(refPath).mtime.toISOString(),
};
} else {
// detached
const date = safeExec(`git log -1 --format="%ci"`, repoPath);
return {
commit: head.substring(0, 7),
fullCommit: head,
detached: true,
lastModified: date ? new Date(date).toISOString() : statSync(headPath).mtime.toISOString(),
};
}
} catch (e: any) {
return { error: e?.message || "git meta error" };
}
}
// Format results to be MCP-tool compatible, keep legacy formatting
async function handleMCPRequest(content: string) {
try {
// Use simple BM25 search with centralized config
const results = await search(content, {
k: CONFIG.RETURN_K
});
if (results.length === 0) {
return {
role: "assistant",
content: `No results found for "${content}". Try searching for UI5 controls like 'button', 'table', 'wizard', testing topics like 'wdi5', 'testing', 'e2e', or concepts like 'routing', 'annotation', 'authentication'.`
};
}
// Format results with URL generation
const formattedResults = results.map((r, index) => {
return formatSearchResult(r, CONFIG.EXCERPT_LENGTH_MAIN, {
generateDocumentationUrl,
getDocUrlConfig
});
}).join('\n');
const summary = `Found ${results.length} results for '${content}':\n\n${formattedResults}`;
return { role: "assistant", content: summary };
} catch (error) {
console.error('Hybrid search failed, falling back to original search:', error);
// Fallback to original search
try {
const searchResult = await searchLibraries(content);
if (searchResult.results.length > 0) {
return { role: "assistant", content: searchResult.results[0].description };
}
return {
role: "assistant",
content: searchResult.error || `No results for "${content}". Try 'button', 'table', 'wizard', 'routing', 'annotation', 'authentication', 'cds entity', 'wdi5 testing'.`,
};
} catch (fallbackError) {
console.error("Search error:", error);
return { role: "assistant", content: `Error searching for "${content}". Try a different query.` };
}
}
}
function json(res: any, code: number, payload: unknown) {
res.statusCode = code;
res.setHeader("Content-Type", "application/json");
res.end(JSON.stringify(payload, null, 2));
}
// ---- server -----------------------------------------------------------------
const server = createServer(async (req, res) => {
// CORS (you can tighten later if needed)
res.setHeader("Access-Control-Allow-Origin", "*");
res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
res.setHeader("Access-Control-Allow-Headers", "Content-Type");
if (req.method === "OPTIONS") return json(res, 200, { ok: true });
// healthz/readyz: cheap checks for PM2/K8s or manual curl
if (req.method === "GET" && (req.url === "/healthz" || req.url === "/readyz")) {
return json(res, 200, { status: "ok", ts: new Date().toISOString() });
}
// status: richer info
if (req.method === "GET" && req.url === "/status") {
// top-level repo git info
let gitInfo: any = {};
try {
const repoPath = resolve(__dirname, "../..");
gitInfo = readGitMeta(repoPath);
// normalize to include branch if unknown
if (!gitInfo.branch) {
const branch = safeExec("git rev-parse --abbrev-ref HEAD", repoPath);
if (branch && branch !== "HEAD") gitInfo.branch = branch;
}
} catch {
gitInfo = { error: "Git info not available" };
}
// docs/search status
const sourcesRoot = join(__dirname, "../../sources");
const knownSources = [
"sapui5-docs",
"cap-docs",
"openui5",
"wdi5",
"ui5-tooling",
"cloud-mta-build-tool",
"ui5-webcomponents",
"cloud-sdk",
"cloud-sdk-ai"
];
const presentSources = existsSync(sourcesRoot)
? readdirSync(sourcesRoot, { withFileTypes: true })
.filter((e) => e.isDirectory())
.map((e) => e.name)
: [];
const toCheck = knownSources.filter((s) => presentSources.includes(s));
const resources: Record<string, any> = {};
let totalResources = 0;
for (const name of knownSources) {
const p = join(sourcesRoot, name);
if (!existsSync(p)) {
resources[name] = { status: "missing", error: "not found" };
continue;
}
const meta = readGitMeta(p);
if ((meta as any).error) {
// still count as available content; just git meta missing (e.g., copied tree)
resources[name] = { status: "available", note: (meta as any).error, path: p };
totalResources++;
} else {
resources[name] = { status: "available", path: p, ...meta };
totalResources++;
}
}
// index + FTS footprint
const dataRoot = join(__dirname, "../../data");
const indexJson = join(dataRoot, "index.json");
const ftsDb = join(dataRoot, "docs.sqlite");
const indexStat = existsSync(indexJson) ? statSync(indexJson) : null;
const ftsStat = existsSync(ftsDb) ? statSync(ftsDb) : null;
// quick search smoke test
let docsStatus = "unknown";
try {
const testSearch = await searchLibraries("button");
docsStatus = testSearch.results.length > 0 ? "available" : "no_results";
} catch {
docsStatus = "error";
}
const statusResponse = {
status: "healthy",
service: packageInfo.name,
version: packageInfo.version,
timestamp: new Date().toISOString(),
buildTimestamp,
git: gitInfo,
documentation: {
status: docsStatus,
searchAvailable: true,
communityAvailable: true,
resources: {
totalResources,
sources: resources,
lastUpdated:
Object.values(resources)
.map((s: any) => s.lastModified)
.filter(Boolean)
.sort()
.pop() || "unknown",
artifacts: {
indexJson: indexStat
? { path: indexJson, sizeBytes: indexStat.size, mtime: indexStat.mtime.toISOString() }
: "missing",
ftsSqlite: ftsStat
? { path: ftsDb, sizeBytes: ftsStat.size, mtime: ftsStat.mtime.toISOString() }
: "missing",
},
},
},
deployment: {
method: process.env.DEPLOYMENT_METHOD || "unknown",
timestamp: process.env.DEPLOYMENT_TIMESTAMP || "unknown",
triggeredBy: process.env.GITHUB_ACTOR || "unknown",
},
runtime: {
uptimeSeconds: process.uptime(),
nodeVersion: process.version,
platform: process.platform,
pid: process.pid,
port: Number(process.env.PORT || 3001),
bind: "127.0.0.1",
},
};
return json(res, 200, statusResponse);
}
// Legacy SSE endpoint - redirect to MCP
if (req.url === "/sse") {
const redirectInfo = {
error: "SSE endpoint deprecated",
message: "The /sse endpoint has been removed. Please use the modern /mcp endpoint instead.",
migration: {
old_endpoint: "/sse",
new_endpoint: "/mcp",
transport: "MCP Streamable HTTP",
protocol_version: "2025-07-09"
},
documentation: "https://github.com/marianfoo/mcp-sap-docs#connect-from-your-mcp-client",
alternatives: {
"Local MCP Streamable HTTP": "http://127.0.0.1:3122/mcp",
"Public MCP Streamable HTTP": "https://mcp-sap-docs.marianzeis.de/mcp"
}
};
res.setHeader("Content-Type", "application/json");
return json(res, 410, redirectInfo);
}
if (req.method === "POST" && req.url === "/mcp") {
let body = "";
req.on("data", (chunk) => (body += chunk.toString()));
req.on("end", async () => {
try {
const mcpRequest: { role: string; content: string } = JSON.parse(body);
const response = await handleMCPRequest(mcpRequest.content);
return json(res, 200, response);
} catch {
return json(res, 400, { error: "Invalid JSON" });
}
});
return;
}
// default 404 JSON (keeps curl|jq friendly)
return json(res, 404, { error: "Not Found", path: req.url, method: req.method });
});
// Initialize search system with metadata
(async () => {
console.log('🔧 Initializing BM25 search system...');
try {
loadMetadata();
console.log('✅ Search system ready with metadata');
} catch (error) {
console.warn('⚠️ Metadata loading failed, using defaults');
console.log('✅ Search system ready');
}
// Start server
const PORT = Number(process.env.PORT || 3001);
// Bind to 127.0.0.1 to keep local-only
server.listen(PORT, "127.0.0.1", () => {
console.log(`📚 HTTP server running on http://127.0.0.1:${PORT} (status: /status, health: /healthz, ready: /readyz)`);
});
})();
```
--------------------------------------------------------------------------------
/src/streamable-http-server.ts:
--------------------------------------------------------------------------------
```typescript
import express, { Request, Response } from "express";
import { randomUUID } from "node:crypto";
import cors from "cors";
import { Server } from "@modelcontextprotocol/sdk/server/index.js";
import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
import {
isInitializeRequest
} from "@modelcontextprotocol/sdk/types.js";
import { logger } from "./lib/logger.js";
import { BaseServerHandler } from "./lib/BaseServerHandler.js";
// Version will be updated by deployment script
const VERSION = "0.3.19";
// Simple in-memory event store for resumability
class InMemoryEventStore {
private events: Map<string, Array<{ eventId: string; message: any }>> = new Map();
private eventCounter = 0;
async storeEvent(streamId: string, message: any): Promise<string> {
const eventId = `event-${this.eventCounter++}`;
if (!this.events.has(streamId)) {
this.events.set(streamId, []);
}
this.events.get(streamId)!.push({ eventId, message });
// Keep only last 100 events per stream to prevent memory issues
const streamEvents = this.events.get(streamId)!;
if (streamEvents.length > 100) {
streamEvents.splice(0, streamEvents.length - 100);
}
return eventId;
}
async replayEventsAfter(lastEventId: string, { send }: { send: (eventId: string, message: any) => Promise<void> }): Promise<string> {
// Find the stream that contains this event ID
for (const [streamId, events] of this.events.entries()) {
const eventIndex = events.findIndex(e => e.eventId === lastEventId);
if (eventIndex !== -1) {
// Replay all events after the specified event ID
for (let i = eventIndex + 1; i < events.length; i++) {
const event = events[i];
await send(event.eventId, event.message);
}
return streamId;
}
}
// If event ID not found, return a new stream ID
return `stream-${randomUUID()}`;
}
}
function createServer() {
const serverOptions: NonNullable<ConstructorParameters<typeof Server>[1]> & {
protocolVersions?: string[];
} = {
protocolVersions: ["2025-07-09"],
capabilities: {
// resources: {}, // DISABLED: Causes 60,000+ resources which breaks Cursor
tools: {} // Enable tools capability
}
};
const srv = new Server({
name: "SAP Docs Streamable HTTP",
description:
"SAP documentation server with Streamable HTTP transport - supports SAPUI5, CAP, wdi5, SAP Community, SAP Help Portal, and ABAP Keyword Documentation integration",
version: VERSION
}, serverOptions);
// Configure server with shared handlers
BaseServerHandler.configureServer(srv);
return srv;
}
async function main() {
// Initialize search system with metadata
BaseServerHandler.initializeMetadata();
const MCP_PORT = process.env.MCP_PORT ? parseInt(process.env.MCP_PORT, 10) : 3122;
// Create Express application
const app = express();
app.use(express.json());
// Configure CORS to expose Mcp-Session-Id header for browser-based clients
app.use(cors({
origin: '*', // Allow all origins - adjust as needed for production
exposedHeaders: ['Mcp-Session-Id']
}));
// Store transports by session ID
const transports: { [sessionId: string]: StreamableHTTPServerTransport } = {};
// Create event store for resumability
const eventStore = new InMemoryEventStore();
// Legacy SSE endpoint - redirect to MCP
app.all('/sse', (req: Request, res: Response) => {
const redirectInfo = {
error: "SSE endpoint deprecated",
message: "The /sse endpoint has been removed. Please use the modern /mcp endpoint instead.",
migration: {
old_endpoint: "/sse",
new_endpoint: "/mcp",
transport: "MCP Streamable HTTP",
protocol_version: "2025-07-09"
},
documentation: "https://github.com/marianfoo/mcp-sap-docs#connect-from-your-mcp-client",
alternatives: {
"Local MCP Streamable HTTP": "http://127.0.0.1:3122/mcp",
"Public MCP Streamable HTTP": "https://mcp-sap-docs.marianzeis.de/mcp"
}
};
res.status(410).json(redirectInfo);
});
// Handle all MCP Streamable HTTP requests (GET, POST, DELETE) on a single endpoint
app.all('/mcp', async (req: Request, res: Response) => {
const requestId = `http_${Date.now()}_${Math.random().toString(36).substr(2, 6)}`;
logger.debug(`Received ${req.method} request to /mcp`, {
requestId,
userAgent: req.headers['user-agent'],
contentLength: req.headers['content-length'],
sessionId: req.headers['mcp-session-id'] as string || 'none'
});
try {
// Check for existing session ID
const sessionId = req.headers['mcp-session-id'] as string;
let transport: StreamableHTTPServerTransport;
if (sessionId && transports[sessionId]) {
// Reuse existing transport
transport = transports[sessionId];
logger.logTransportEvent('transport_reused', sessionId, {
requestId,
method: req.method,
transportCount: Object.keys(transports).length
});
} else if (!sessionId && req.method === 'POST' && req.is('application/json') && req.body?.method === 'initialize') {
// New initialization request - create new transport
const cleanupTransport = (
sessionId: string | undefined,
trigger: "onsessionclosed" | "onclose",
context: Record<string, unknown> = {}
) => {
if (!sessionId) {
return;
}
const hadTransport = Boolean(transports[sessionId]);
if (hadTransport) {
delete transports[sessionId];
}
logger.logTransportEvent("session_closed", sessionId, {
...context,
trigger,
transportCount: Object.keys(transports).length,
...(hadTransport ? {} : { note: "session already cleaned up" })
});
};
transport = new StreamableHTTPServerTransport({
sessionIdGenerator: () => randomUUID(),
eventStore, // Enable resumability
onsessioninitialized: (sessionId: string) => {
// Store the transport by session ID when session is initialized
logger.logTransportEvent('session_initialized', sessionId, {
requestId,
transportCount: Object.keys(transports).length + 1
});
transports[sessionId] = transport;
},
onsessionclosed: (sessionId: string) => {
cleanupTransport(sessionId, 'onsessionclosed');
}
});
// Set up onclose handler to clean up transport when closed
transport.onclose = () => {
cleanupTransport(transport.sessionId, 'onclose', { requestId });
};
// Connect the transport to the MCP server
const server = createServer();
await server.connect(transport);
logger.logTransportEvent('transport_created', undefined, {
requestId,
method: req.method
});
} else {
// Invalid request - no session ID or not initialization request
logger.warn('Invalid MCP request', {
requestId,
method: req.method,
hasSessionId: !!sessionId,
isInitRequest: req.method === 'POST' && req.is('application/json') && req.body?.method === 'initialize',
sessionId: sessionId || 'none',
userAgent: req.headers['user-agent']
});
res.status(400).json({
jsonrpc: '2.0',
error: {
code: -32000,
message: 'Bad Request: No valid session ID provided or not an initialization request',
},
id: null,
});
return;
}
// Handle the request with the transport
await transport.handleRequest(req, res, req.body);
} catch (error) {
logger.error('Error handling MCP request', {
requestId,
error: String(error),
stack: error instanceof Error ? error.stack : undefined,
method: req.method,
sessionId: req.headers['mcp-session-id'] as string || 'none',
userAgent: req.headers['user-agent']
});
if (!res.headersSent) {
res.status(500).json({
jsonrpc: '2.0',
error: {
code: -32603,
message: `Internal server error. Request ID: ${requestId}`,
},
id: null,
});
}
}
});
// Health check endpoint
app.get('/health', (req: Request, res: Response) => {
res.json({
status: 'healthy',
service: 'mcp-sap-docs-streamable',
version: VERSION,
timestamp: new Date().toISOString(),
transport: 'streamable-http',
protocol: '2025-07-09'
});
});
// Start the server (bind to localhost for local-only access)
const server = app.listen(MCP_PORT, '127.0.0.1', (error?: Error) => {
if (error) {
console.error('Failed to start server:', error);
process.exit(1);
}
});
// Configure server timeouts for MCP connections
server.timeout = 0; // Disable HTTP timeout for long-lived MCP connections
server.keepAliveTimeout = 0; // Disable keep-alive timeout
server.headersTimeout = 0; // Disable headers timeout
console.log(`📚 MCP Streamable HTTP Server listening on http://127.0.0.1:${MCP_PORT}`);
console.log(`
==============================================
MCP STREAMABLE HTTP SERVER
Protocol version: 2025-07-09
Endpoint: /mcp
Methods: GET, POST, DELETE
Usage:
- Initialize with POST to /mcp
- Establish stream with GET to /mcp
- Send requests with POST to /mcp
- Terminate session with DELETE to /mcp
Health check: GET /health
==============================================
`);
// Log server startup
logger.info("MCP SAP Docs Streamable HTTP server starting up", {
port: MCP_PORT,
nodeEnv: process.env.NODE_ENV,
logLevel: process.env.LOG_LEVEL,
logFormat: process.env.LOG_FORMAT
});
// Log successful startup
logger.info("MCP SAP Docs Streamable HTTP server ready", {
transport: "streamable-http",
port: MCP_PORT,
pid: process.pid
});
// Set up performance monitoring (every 5 minutes)
const performanceInterval = setInterval(() => {
logger.logPerformanceMetrics();
logger.info('Active sessions status', {
activeSessions: Object.keys(transports).length,
sessionIds: Object.keys(transports),
timestamp: new Date().toISOString()
});
}, 5 * 60 * 1000);
// Handle server shutdown
process.on('SIGINT', async () => {
logger.info('Shutdown signal received, closing server gracefully');
// Clear performance monitoring
clearInterval(performanceInterval);
// Close all active transports to properly clean up resources
const sessionIds = Object.keys(transports);
logger.info(`Closing ${sessionIds.length} active sessions`);
for (const sessionId of sessionIds) {
try {
logger.logTransportEvent('session_shutdown', sessionId);
await transports[sessionId].close();
delete transports[sessionId];
} catch (error) {
logger.error('Error closing transport during shutdown', {
sessionId,
error: String(error)
});
}
}
logger.info('Server shutdown complete');
process.exit(0);
});
}
main().catch((e) => {
console.error("Fatal:", e);
process.exit(1);
});
```
--------------------------------------------------------------------------------
/test/community-search.ts:
--------------------------------------------------------------------------------
```typescript
#!/usr/bin/env node
// Combined test script for SAP Community Search functionality
// Tests search, batch retrieval, and convenience functions
import {
searchCommunityBestMatch,
getCommunityPostByUrl,
getCommunityPostsByIds,
getCommunityPostById,
searchAndGetTopPosts
} from '../dist/src/lib/communityBestMatch.js';
interface TestOptions {
userAgent: string;
delay: number;
}
const defaultOptions: TestOptions = {
userAgent: 'SAP-Docs-MCP-Test/1.0',
delay: 2000
};
// Utility function for delays
const delay = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
// Test 1: Community Search with HTML Scraping
async function testCommunitySearch(options: TestOptions = defaultOptions): Promise<void> {
console.log('🔍 Testing SAP Community Search with HTML Scraping');
console.log('='.repeat(60));
const testQueries = [
'odata cache',
'fiori elements',
'CAP authentication'
];
for (const query of testQueries) {
console.log(`\n📝 Testing query: "${query}"`);
console.log('-'.repeat(40));
try {
const results = await searchCommunityBestMatch(query, {
includeBlogs: true,
limit: 5,
userAgent: options.userAgent
});
if (results.length === 0) {
console.log('❌ No results found');
continue;
}
console.log(`✅ Found ${results.length} results:`);
results.forEach((result, index) => {
console.log(`\n${index + 1}. ${result.title}`);
console.log(` URL: ${result.url}`);
console.log(` Author: ${result.author || 'Unknown'}`);
console.log(` Published: ${result.published || 'Unknown'}`);
console.log(` Likes: ${result.likes || 0}`);
console.log(` Snippet: ${result.snippet ? result.snippet.substring(0, 100) + '...' : 'No snippet'}`);
console.log(` Tags: ${result.tags?.join(', ') || 'None'}`);
console.log(` Post ID: ${result.postId || 'Not extracted'}`);
// Verify post ID extraction
if (result.postId) {
console.log(` ✅ Post ID extracted: ${result.postId}`);
} else {
console.log(` ⚠️ Post ID not extracted from URL: ${result.url}`);
}
});
// Test detailed post retrieval for the first result using URL scraping
if (results.length > 0) {
console.log(`\n🔎 Testing URL-based post retrieval for: "${results[0].title}"`);
console.log('-'.repeat(30));
try {
const postContent = await getCommunityPostByUrl(results[0].url, options.userAgent);
if (postContent) {
console.log('✅ Successfully retrieved full post content via URL scraping:');
console.log(postContent.substring(0, 400) + '...\n');
} else {
console.log('❌ Failed to retrieve full post content via URL scraping');
}
} catch (error: any) {
console.log(`❌ Error retrieving post content: ${error.message}`);
}
}
} catch (error: any) {
console.log(`❌ Error searching for "${query}": ${error.message}`);
}
// Add delay between requests to be respectful
if (options.delay > 0) {
await delay(options.delay);
}
}
}
// Test 2: Batch Retrieval with LiQL API
async function testBatchRetrieval(options: TestOptions = defaultOptions): Promise<void> {
console.log('\n\n📦 Testing SAP Community Batch Retrieval with LiQL API');
console.log('='.repeat(60));
// Test with known post IDs
const testPostIds = ['13961398', '13446100', '14152848'];
console.log('📦 Testing batch retrieval for multiple posts:');
console.log(`Post IDs: ${testPostIds.join(', ')}`);
console.log('-'.repeat(40));
try {
const results = await getCommunityPostsByIds(testPostIds, options.userAgent);
console.log(`✅ Successfully retrieved ${Object.keys(results).length} out of ${testPostIds.length} posts\n`);
for (const postId of testPostIds) {
if (results[postId]) {
console.log(`✅ Post ${postId}:`);
const content = results[postId];
const lines = content.split('\n');
console.log(` Title: ${lines[0].replace('# ', '')}`);
// Extract published date
const publishedLine = lines.find(line => line.startsWith('**Published**:'));
if (publishedLine) {
console.log(` ${publishedLine}`);
}
// Show content preview
const contentStart = content.indexOf('---\n\n') + 5;
const contentEnd = content.lastIndexOf('\n\n---');
if (contentStart > 4 && contentEnd > contentStart) {
const contentPreview = content.slice(contentStart, contentEnd).substring(0, 200) + '...';
console.log(` Content preview: ${contentPreview}`);
}
console.log();
} else {
console.log(`❌ Post ${postId}: Not retrieved`);
}
}
} catch (error: any) {
console.log(`❌ Batch retrieval failed: ${error.message}`);
}
}
// Test 3: Single Post Retrieval
async function testSingleRetrieval(options: TestOptions = defaultOptions): Promise<void> {
console.log('\n🎯 Testing single post retrieval via LiQL API');
console.log('='.repeat(60));
const testPostId = '13961398'; // FIORI Cache Maintenance
try {
console.log(`Testing single retrieval for post: ${testPostId}`);
const content = await getCommunityPostById(testPostId, options.userAgent);
if (content) {
console.log('✅ Successfully retrieved single post:');
console.log(content.substring(0, 500) + '...\n');
// Verify expected content
if (content.includes('FIORI Cache Maintenance')) {
console.log('✅ Title verification successful');
}
if (content.includes('SMICM')) {
console.log('✅ Content verification successful');
}
if (content.includes('2024')) {
console.log('✅ Date verification successful');
}
} else {
console.log('❌ Single retrieval failed - no content returned');
}
} catch (error: any) {
console.log(`❌ Single retrieval failed: ${error.message}`);
}
}
// Test 4: Direct LiQL API Testing
async function testLiQLAPIDirectly(options: TestOptions = defaultOptions): Promise<void> {
console.log('\n\n🧪 Testing LiQL API directly');
console.log('='.repeat(60));
const testIds = ['13961398', '13446100'];
const idList = testIds.map(id => `'${id}'`).join(', ');
const liqlQuery = `select body, id, subject, search_snippet, post_time from messages where id in (${idList})`;
const url = `https://community.sap.com/api/2.0/search?q=${encodeURIComponent(liqlQuery)}`;
console.log(`Testing URL: ${url.substring(0, 120)}...`);
try {
const response = await fetch(url, {
headers: {
'Accept': 'application/json',
'User-Agent': options.userAgent
}
});
if (!response.ok) {
console.log(`❌ API returned ${response.status}: ${response.statusText}`);
return;
}
const data = await response.json();
console.log(`✅ API Response status: ${data.status}`);
console.log(`✅ Items returned: ${data.data?.items?.length || 0}`);
if (data.data?.items) {
for (const item of data.data.items) {
console.log(` - Post ${item.id}: "${item.subject}" (${item.post_time})`);
}
}
} catch (error: any) {
console.log(`❌ Direct API test failed: ${error.message}`);
}
}
// Test 5: Convenience Function (Search + Get Top Posts)
async function testConvenienceFunction(options: TestOptions = defaultOptions): Promise<void> {
console.log('\n\n🚀 Testing Search + Get Top Posts Convenience Function');
console.log('='.repeat(60));
const query = 'odata cache';
const topN = 3;
console.log(`Query: "${query}"`);
console.log(`Getting top ${topN} posts with full content...\n`);
try {
const result = await searchAndGetTopPosts(query, topN, {
includeBlogs: true,
userAgent: options.userAgent
});
console.log(`✅ Search found ${result.search.length} results`);
console.log(`✅ Retrieved full content for ${Object.keys(result.posts).length} posts\n`);
// Display search results with post content
for (let i = 0; i < result.search.length; i++) {
const searchResult = result.search[i];
const postContent = result.posts[searchResult.postId || ''];
console.log(`${i + 1}. ${searchResult.title}`);
console.log(` Post ID: ${searchResult.postId}`);
console.log(` URL: ${searchResult.url}`);
console.log(` Author: ${searchResult.author || 'Unknown'}`);
console.log(` Likes: ${searchResult.likes || 0}`);
if (postContent) {
console.log(' ✅ Full content retrieved:');
const contentPreview = postContent.split('\n\n---\n\n')[1] || postContent;
console.log(` "${contentPreview.substring(0, 150)}..."`);
} else {
console.log(' ❌ Full content not available');
}
console.log();
}
// Example usage demonstration
console.log('📋 Example: How to use this data');
console.log('='.repeat(40));
console.log('// Search and get top 3 posts about OData cache:');
console.log(`const { search, posts } = await searchAndGetTopPosts('${query}', ${topN});`);
console.log('');
console.log('// Display results:');
console.log('search.forEach((result, index) => {');
console.log(' console.log(`${index + 1}. ${result.title}`);');
console.log(' if (posts[result.postId]) {');
console.log(' console.log(posts[result.postId]); // Full formatted content');
console.log(' }');
console.log('});');
} catch (error: any) {
console.error(`❌ Test failed: ${error.message}`);
}
}
// Test 6: Specific Known Post
async function testSpecificPost(options: TestOptions = defaultOptions): Promise<void> {
console.log('\n\n🎯 Testing specific known post retrieval');
console.log('='.repeat(60));
// Test with the known SAP Community URL
const testUrl = 'https://community.sap.com/t5/technology-blog-posts-by-sap/fiori-cache-maintenance/ba-p/13961398';
try {
console.log(`Testing URL: ${testUrl}`);
console.log(`Expected Post ID: 13961398`);
const content = await getCommunityPostByUrl(testUrl, options.userAgent);
if (content) {
console.log('✅ Successfully retrieved content:');
console.log(content.substring(0, 600) + '...');
// Verify the content contains expected elements
if (content.includes('FIORI Cache Maintenance')) {
console.log('✅ Title extraction successful');
}
if (content.includes('MarkNed')) {
console.log('✅ Author extraction successful');
}
if (content.includes('SMICM')) {
console.log('✅ Content extraction successful');
}
} else {
console.log('❌ No content retrieved');
}
} catch (error: any) {
console.log(`❌ Error: ${error.message}`);
}
}
// Main test runner
async function main(): Promise<void> {
console.log('🚀 SAP Community Search - Comprehensive Test Suite');
console.log('==================================================');
console.log(`Started at: ${new Date().toISOString()}\n`);
const options: TestOptions = {
userAgent: 'SAP-Docs-MCP-Test/1.0',
delay: 1500 // Reduced delay for faster testing
};
try {
// Run all tests sequentially
await testCommunitySearch(options);
await testBatchRetrieval(options);
await testSingleRetrieval(options);
await testLiQLAPIDirectly(options);
await testConvenienceFunction(options);
await testSpecificPost(options);
console.log('\n\n🎉 All tests completed successfully!');
console.log('=====================================');
console.log(`Finished at: ${new Date().toISOString()}`);
} catch (error: any) {
console.error('❌ Test suite failed:', error);
process.exit(1);
}
}
// Handle graceful shutdown
process.on('SIGINT', () => {
console.log('\n👋 Test interrupted by user');
process.exit(0);
});
// Run the tests if this file is executed directly
if (import.meta.url === `file://${process.argv[1]}`) {
main().catch(error => {
console.error('💥 Unexpected error:', error);
process.exit(1);
});
}
// Export functions for potential use in other test files
export {
testCommunitySearch,
testBatchRetrieval,
testSingleRetrieval,
testLiQLAPIDirectly,
testConvenienceFunction,
testSpecificPost,
main as runAllTests
};
```
--------------------------------------------------------------------------------
/src/lib/communityBestMatch.ts:
--------------------------------------------------------------------------------
```typescript
// src/lib/communityBestMatch.ts
// Scrape SAP Community search "Best Match" results directly from the HTML page.
// No external dependencies; best-effort selectors based on current Khoros layout.
import { CONFIG } from "./config.js";
import { truncateContent } from "./truncate.js";
export interface BestMatchHit {
title: string;
url: string;
author?: string;
published?: string; // e.g., "2024 Dec 11 4:31 PM"
likes?: number;
snippet?: string;
tags?: string[];
postId?: string; // extracted from URL for retrieval
}
type Options = {
includeBlogs?: boolean; // default true
limit?: number; // default 20
userAgent?: string; // optional UA override
};
const BASE = "https://community.sap.com";
const buildSearchUrl = (q: string, includeBlogs = true) => {
const params = new URLSearchParams({
collapse_discussion: "true",
q,
});
if (includeBlogs) {
params.set("filter", "includeBlogs");
params.set("include_blogs", "true");
}
// "tab/message" view surfaces posts sorted by Best Match by default
return `${BASE}/t5/forums/searchpage/tab/message?${params.toString()}`;
};
const decodeEntities = (s = "") =>
s
.replace(/&/g, "&")
.replace(/</g, "<")
.replace(/>/g, ">")
.replace(/"/g, '"')
.replace(/'/g, "'");
const stripTags = (html = "") =>
decodeEntities(html.replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim());
const absolutize = (href: string) =>
href?.startsWith("http") ? href : new URL(href, BASE).href;
// Extract post ID from URL for later retrieval
const extractPostId = (url: string): string | undefined => {
// Extract from URL patterns like: /ba-p/13961398 or /td-p/13961398
const urlMatch = url.match(/\/(?:ba-p|td-p)\/(\d+)/);
if (urlMatch) {
return urlMatch[1];
}
// Fallback: extract from end of URL
const endMatch = url.match(/\/(\d+)(?:\?|$)/);
return endMatch ? endMatch[1] : undefined;
};
async function fetchText(url: string, userAgent?: string) {
const res = await fetch(url, {
headers: {
"User-Agent": userAgent || "sap-docs-mcp/1.0 (BestMatchScraper)",
"Accept": "text/html,application/xhtml+xml",
},
});
if (!res.ok) throw new Error(`${url} -> ${res.status} ${res.statusText}`);
return res.text();
}
function parseHitsFromHtml(html: string, limit = 20): BestMatchHit[] {
const results: BestMatchHit[] = [];
// Find all message wrapper divs with data-lia-message-uid
const wrapperRegex = /<div[^>]+data-lia-message-uid="([^"]*)"[^>]*class="[^"]*lia-message-view-wrapper[^"]*"[^>]*>([\s\S]*?)(?=<div[^>]+class="[^"]*lia-message-view-wrapper|$)/gi;
let match;
while ((match = wrapperRegex.exec(html)) !== null && results.length < limit) {
const postId = match[1];
const seg = match[2].slice(0, 60000); // safety cap
// Title + URL
const titleMatch =
seg.match(
/<h2[^>]*class="[^"]*message-subject[^"]*"[^>]*>[\s\S]*?<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i
) ||
seg.match(
/<a[^>]+class="page-link[^"]*"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i
);
const url = titleMatch ? absolutize(decodeEntities(titleMatch[1])) : "";
const title = titleMatch ? stripTags(titleMatch[2]) : "";
if (!title || !url) continue;
// Author
// Look for "View Profile of ..." or the user link block
let author = "";
const authorMatch =
seg.match(/viewprofilepage\/user-id\/\d+[^>]*>([^<]+)/i) ||
seg.match(/class="[^"]*lia-user-name-link[^"]*"[^>]*>([^<]+)/i);
if (authorMatch) author = stripTags(authorMatch[1]);
// Date/time
const dateMatch = seg.match(/class="local-date"[^>]*>([^<]+)</i);
const timeMatch = seg.match(/class="local-time"[^>]*>([^<]+)</i);
const published = dateMatch
? `${stripTags(dateMatch[1])}${timeMatch ? " " + stripTags(timeMatch[1]) : ""}`
: undefined;
// Likes (Kudos)
const likesMatch = seg.match(/Kudos Count\s+(\d+)/i);
const likes = likesMatch ? Number(likesMatch[1]) : undefined;
// Snippet
const snippetMatch = seg.match(
/<div[^>]*class="[^"]*lia-truncated-body-container[^"]*"[^>]*>([\s\S]*?)<\/div>/i
);
const snippet = snippetMatch ? stripTags(snippetMatch[1]).slice(0, CONFIG.EXCERPT_LENGTH_COMMUNITY) : undefined;
// Tags
const tagSectionMatch = seg.match(
/<div[^>]*class="[^"]*TagList[^"]*"[^>]*>[\s\S]*?<\/div>/i
);
const tags: string[] = [];
if (tagSectionMatch) {
const tagLinks = tagSectionMatch[0].matchAll(
/<a[^>]*class="[^"]*lia-tag[^"]*"[^>]*>([\s\S]*?)<\/a>/gi
);
for (const m of tagLinks) {
const t = stripTags(m[1]);
if (t) tags.push(t);
}
}
results.push({ title, url, author, published, likes, snippet, tags, postId });
}
return results;
}
export async function searchCommunityBestMatch(
query: string,
opts: Options = {}
): Promise<BestMatchHit[]> {
const { includeBlogs = true, limit = 20, userAgent } = opts;
const url = buildSearchUrl(query, includeBlogs);
const html = await fetchText(url, userAgent);
return parseHitsFromHtml(html, limit);
}
// Convenience function: Search and get full content of top N posts in one call
export async function searchAndGetTopPosts(
query: string,
topN: number = 3,
opts: Options = {}
): Promise<{ search: BestMatchHit[], posts: { [id: string]: string } }> {
// First, search for posts
const searchResults = await searchCommunityBestMatch(query, { ...opts, limit: Math.max(topN, opts.limit || 20) });
// Extract post IDs from top N results
const topResults = searchResults.slice(0, topN);
const postIds = topResults
.map(result => result.postId)
.filter((id): id is string => id !== undefined);
// Batch retrieve full content
const posts = await getCommunityPostsByIds(postIds, opts.userAgent);
return {
search: topResults,
posts
};
}
// Function to get full post content by scraping the post page
// Batch retrieve multiple posts using LiQL API
export async function getCommunityPostsByIds(postIds: string[], userAgent?: string): Promise<{ [id: string]: string }> {
const results: { [id: string]: string } = {};
if (postIds.length === 0) {
return results;
}
try {
// Build LiQL query for batch retrieval
const idList = postIds.map(id => `'${id}'`).join(', ');
const liqlQuery = `
select body, id, subject, search_snippet, post_time, view_href
from messages
where id in (${idList})
`.replace(/\s+/g, ' ').trim();
const url = `https://community.sap.com/api/2.0/search?q=${encodeURIComponent(liqlQuery)}`;
const response = await fetch(url, {
headers: {
'Accept': 'application/json',
'User-Agent': userAgent || 'sap-docs-mcp/1.0 (BatchRetrieval)'
}
});
if (!response.ok) {
console.warn(`SAP Community API returned ${response.status}: ${response.statusText}`);
return results;
}
const data = await response.json() as any;
if (data.status !== 'success' || !data.data?.items) {
return results;
}
// Process each post
for (const post of data.data.items) {
const postDate = post.post_time ? new Date(post.post_time).toLocaleDateString() : 'Unknown';
const postUrl = post.view_href || `https://community.sap.com/t5/technology-blogs-by-sap/bg-p/t/${post.id}`;
const fullContent = `# ${post.subject}
**Source**: SAP Community Blog Post
**Published**: ${postDate}
**URL**: ${postUrl}
---
${post.body || post.search_snippet}
---
*This content is from the SAP Community and represents community knowledge and experiences.*`;
// Apply intelligent truncation if content is too large
const truncationResult = truncateContent(fullContent);
results[post.id] = truncationResult.content;
}
return results;
} catch (error) {
console.warn('Failed to batch retrieve community posts:', error);
return results;
}
}
// Single post retrieval using LiQL API
export async function getCommunityPostById(postId: string, userAgent?: string): Promise<string | null> {
const results = await getCommunityPostsByIds([postId], userAgent);
return results[postId] || null;
}
export async function getCommunityPostByUrl(postUrl: string, userAgent?: string): Promise<string | null> {
try {
const html = await fetchText(postUrl, userAgent);
// Extract title - try multiple selectors
let title = "Untitled";
const titleSelectors = [
/<h1[^>]*class="[^"]*lia-message-subject[^"]*"[^>]*>([\s\S]*?)<\/h1>/i,
/<h2[^>]*class="[^"]*message-subject[^"]*"[^>]*>([\s\S]*?)<\/h2>/i,
/<title>([\s\S]*?)<\/title>/i
];
for (const selector of titleSelectors) {
const titleMatch = html.match(selector);
if (titleMatch) {
title = stripTags(titleMatch[1]).replace(/\s*-\s*SAP Community.*$/, '').trim();
break;
}
}
// Extract author and date - multiple patterns
let author = "Unknown";
const authorSelectors = [
/class="[^"]*lia-user-name-link[^"]*"[^>]*>([^<]+)/i,
/viewprofilepage\/user-id\/\d+[^>]*>([^<]+)/i,
/"author"[^>]*>[\s\S]*?<[^>]*>([^<]+)/i
];
for (const selector of authorSelectors) {
const authorMatch = html.match(selector);
if (authorMatch) {
author = stripTags(authorMatch[1]);
break;
}
}
// Extract date and time
const dateMatch = html.match(/class="local-date"[^>]*>([^<]+)</i);
const timeMatch = html.match(/class="local-time"[^>]*>([^<]+)</i);
const published = dateMatch
? `${stripTags(dateMatch[1])}${timeMatch ? " " + stripTags(timeMatch[1]) : ""}`
: "Unknown";
// Extract main content - try multiple content selectors
let content = "Content not available";
const contentSelectors = [
/<div[^>]*class="[^"]*lia-message-body[^"]*"[^>]*>([\s\S]*?)<\/div>/i,
/<div[^>]*class="[^"]*lia-message-body-content[^"]*"[^>]*>([\s\S]*?)<\/div>/i,
/<div[^>]*class="[^"]*messageBody[^"]*"[^>]*>([\s\S]*?)<\/div>/i
];
for (const selector of contentSelectors) {
const contentMatch = html.match(selector);
if (contentMatch) {
// Clean up the content - remove script tags, preserve some formatting
let rawContent = contentMatch[1]
.replace(/<script[\s\S]*?<\/script>/gi, '')
.replace(/<style[\s\S]*?<\/style>/gi, '')
.replace(/<iframe[\s\S]*?<\/iframe>/gi, '[Embedded Content]');
// Convert some HTML elements to markdown-like format
rawContent = rawContent
.replace(/<h([1-6])[^>]*>([\s\S]*?)<\/h[1-6]>/gi, (_, level, text) => {
const hashes = '#'.repeat(parseInt(level) + 1);
return `\n${hashes} ${stripTags(text)}\n`;
})
.replace(/<p[^>]*>([\s\S]*?)<\/p>/gi, '\n$1\n')
.replace(/<br\s*\/?>/gi, '\n')
.replace(/<strong[^>]*>([\s\S]*?)<\/strong>/gi, '**$1**')
.replace(/<em[^>]*>([\s\S]*?)<\/em>/gi, '*$1*')
.replace(/<code[^>]*>([\s\S]*?)<\/code>/gi, '`$1`')
.replace(/<pre[^>]*>([\s\S]*?)<\/pre>/gi, '\n```\n$1\n```\n')
.replace(/<ul[^>]*>([\s\S]*?)<\/ul>/gi, '$1')
.replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, '- $1\n');
content = stripTags(rawContent).replace(/\n\s*\n\s*\n/g, '\n\n').trim();
break;
}
}
// Extract tags
const tagSectionMatch = html.match(
/<div[^>]*class="[^"]*TagList[^"]*"[^>]*>[\s\S]*?<\/div>/i
);
const tags: string[] = [];
if (tagSectionMatch) {
const tagLinks = tagSectionMatch[0].matchAll(
/<a[^>]*class="[^"]*lia-tag[^"]*"[^>]*>([\s\S]*?)<\/a>/gi
);
for (const m of tagLinks) {
const t = stripTags(m[1]);
if (t) tags.push(t);
}
}
// Extract kudos count
let kudos = 0;
const kudosMatch = html.match(/(\d+)\s+Kudos?/i);
if (kudosMatch) {
kudos = parseInt(kudosMatch[1]);
}
const tagsText = tags.length > 0 ? `\n**Tags:** ${tags.join(", ")}` : "";
const kudosText = kudos > 0 ? `\n**Kudos:** ${kudos}` : "";
const fullContent = `# ${title}
**Source**: SAP Community Blog Post
**Author**: ${author}
**Published**: ${published}${kudosText}${tagsText}
**URL**: ${postUrl}
---
${content}
---
*This content is from the SAP Community and represents community knowledge and experiences.*`;
// Apply intelligent truncation if content is too large
const truncationResult = truncateContent(fullContent);
return truncationResult.content;
} catch (error) {
console.warn('Failed to get community post:', error);
return null;
}
}
```