This is page 3 of 5. Use http://codebase.md/marianfoo/mcp-sap-docs?lines=true&page={x} to view the full context.
# Directory Structure
```
├── .cursor
│ └── rules
│ ├── 00-overview.mdc
│ ├── 10-search-stack.mdc
│ ├── 20-tools-and-apis.mdc
│ ├── 30-tests-and-output.mdc
│ ├── 40-deploy.mdc
│ ├── 50-metadata-config.mdc
│ ├── 60-adding-github-sources.mdc
│ ├── 70-tool-usage-guide.mdc
│ └── 80-abap-integration.mdc
├── .cursorignore
├── .gitattributes
├── .github
│ ├── ISSUE_TEMPLATE
│ │ ├── config.yml
│ │ ├── missing-documentation.yml
│ │ └── new-documentation-source.yml
│ └── workflows
│ ├── deploy-mcp-sap-docs.yml
│ ├── test-pr.yml
│ └── update-submodules.yml
├── .gitignore
├── .gitmodules
├── .npmignore
├── .vscode
│ ├── extensions.json
│ └── settings.json
├── docs
│ ├── ABAP-INTEGRATION-SUMMARY.md
│ ├── ABAP-MULTI-VERSION-INTEGRATION.md
│ ├── ABAP-STANDARD-INTEGRATION.md
│ ├── ABAP-USAGE-GUIDE.md
│ ├── ARCHITECTURE.md
│ ├── COMMUNITY-SEARCH-IMPLEMENTATION.md
│ ├── CONTENT-SIZE-LIMITS.md
│ ├── CURSOR-SETUP.md
│ ├── DEV.md
│ ├── FTS5-IMPLEMENTATION-COMPLETE.md
│ ├── LLM-FRIENDLY-IMPROVEMENTS.md
│ ├── METADATA-CONSOLIDATION.md
│ ├── TEST-SEARCH.md
│ └── TESTS.md
├── ecosystem.config.cjs
├── index.html
├── LICENSE
├── package-lock.json
├── package.json
├── README.md
├── REMOTE_SETUP.md
├── scripts
│ ├── build-fts.ts
│ ├── build-index.ts
│ ├── check-version.js
│ └── summarize-src.js
├── server.json
├── setup.sh
├── src
│ ├── global.d.ts
│ ├── http-server.ts
│ ├── lib
│ │ ├── BaseServerHandler.ts
│ │ ├── communityBestMatch.ts
│ │ ├── config.ts
│ │ ├── localDocs.ts
│ │ ├── logger.ts
│ │ ├── metadata.ts
│ │ ├── sapHelp.ts
│ │ ├── search.ts
│ │ ├── searchDb.ts
│ │ ├── truncate.ts
│ │ ├── types.ts
│ │ └── url-generation
│ │ ├── abap.ts
│ │ ├── BaseUrlGenerator.ts
│ │ ├── cap.ts
│ │ ├── cloud-sdk.ts
│ │ ├── dsag.ts
│ │ ├── GenericUrlGenerator.ts
│ │ ├── index.ts
│ │ ├── README.md
│ │ ├── sapui5.ts
│ │ ├── utils.ts
│ │ └── wdi5.ts
│ ├── metadata.json
│ ├── server.ts
│ └── streamable-http-server.ts
├── test
│ ├── _utils
│ │ ├── httpClient.js
│ │ └── parseResults.js
│ ├── community-search.ts
│ ├── comprehensive-url-generation.test.ts
│ ├── performance
│ │ └── README.md
│ ├── prompts.test.ts
│ ├── quick-url-test.ts
│ ├── README.md
│ ├── tools
│ │ ├── run-tests.js
│ │ ├── sap_docs_search
│ │ │ ├── search-cap-docs.js
│ │ │ ├── search-cloud-sdk-ai.js
│ │ │ ├── search-cloud-sdk-js.js
│ │ │ └── search-sapui5-docs.js
│ │ ├── search-url-verification.js
│ │ ├── search.generic.spec.js
│ │ └── search.smoke.js
│ ├── url-status.ts
│ └── validate-urls.ts
├── test-community-search.js
├── test-search-interactive.ts
├── test-search.http
├── test-search.ts
├── tsconfig.json
└── vitest.config.ts
```
# Files
--------------------------------------------------------------------------------
/src/lib/sapHelp.ts:
--------------------------------------------------------------------------------
```typescript
1 | import {
2 | SearchResponse,
3 | SearchResult,
4 | SapHelpSearchResponse,
5 | SapHelpMetadataResponse,
6 | SapHelpPageContentResponse
7 | } from "./types.js";
8 | import { truncateContent } from "./truncate.js";
9 |
10 | const BASE = "https://help.sap.com";
11 |
12 | // ---------- Utils ----------
13 | function toQuery(params: Record<string, any>): string {
14 | return Object.entries(params)
15 | .filter(([, v]) => v !== undefined && v !== null && v !== "")
16 | .map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(String(v))}`)
17 | .join("&");
18 | }
19 |
20 | function ensureAbsoluteUrl(url: string): string {
21 | if (url.startsWith('http://') || url.startsWith('https://')) {
22 | return url;
23 | }
24 | // Ensure leading slash for relative URLs
25 | const cleanUrl = url.startsWith('/') ? url : '/' + url;
26 | return BASE + cleanUrl;
27 | }
28 |
29 | function parseDocsPathParts(urlOrPath: string): { productUrlSeg: string; deliverableLoio: string } {
30 | // Accept relative path like /docs/PROD/DELIVERABLE/FILE.html?... or full URL
31 | const u = new URL(urlOrPath, BASE);
32 | const parts = u.pathname.split("/").filter(Boolean); // ["docs", "{product}", "{deliverable}", "{file}.html"]
33 | if (parts[0] !== "docs" || parts.length < 4) {
34 | throw new Error("Unexpected docs URL: " + u.href);
35 | }
36 | const productUrlSeg = parts[1];
37 | const deliverableLoio = parts[2]; // e.g., 007d655fd353410e9bbba4147f56c2f0
38 | return { productUrlSeg, deliverableLoio };
39 | }
40 |
41 | /**
42 | * Search SAP Help using the private elasticsearch endpoint
43 | */
44 | export async function searchSapHelp(query: string): Promise<SearchResponse> {
45 | try {
46 | const searchParams = {
47 | transtype: "standard,html,pdf,others",
48 | state: "PRODUCTION,TEST,DRAFT",
49 | product: "",
50 | version: "",
51 | q: query,
52 | to: "19", // Limit to 20 results (0-19)
53 | area: "content",
54 | advancedSearch: "0",
55 | excludeNotSearchable: "1",
56 | language: "en-US",
57 | };
58 |
59 | const searchUrl = `${BASE}/http.svc/elasticsearch?${toQuery(searchParams)}`;
60 |
61 | const response = await fetch(searchUrl, {
62 | headers: {
63 | Accept: "application/json",
64 | "User-Agent": "mcp-sap-docs/help-search",
65 | Referer: BASE,
66 | },
67 | });
68 |
69 | if (!response.ok) {
70 | throw new Error(`SAP Help search failed: ${response.status} ${response.statusText}`);
71 | }
72 |
73 | const data: SapHelpSearchResponse = await response.json();
74 | const results = data?.data?.results || [];
75 |
76 | if (!results.length) {
77 | return {
78 | results: [],
79 | error: `No SAP Help results found for "${query}"`
80 | };
81 | }
82 |
83 | // Store the search results for later retrieval
84 | const searchResults: SearchResult[] = results.map((hit, index) => ({
85 | library_id: `sap-help-${hit.loio}`,
86 | topic: '',
87 | id: `sap-help-${hit.loio}`,
88 | title: hit.title,
89 | url: ensureAbsoluteUrl(hit.url),
90 | snippet: `${hit.snippet || hit.title} — Product: ${hit.product || hit.productId || "Unknown"} (${hit.version || hit.versionId || "Latest"})`,
91 | score: 0,
92 | metadata: {
93 | source: "help",
94 | loio: hit.loio,
95 | product: hit.product || hit.productId,
96 | version: hit.version || hit.versionId,
97 | rank: index + 1
98 | },
99 | // Legacy fields for backward compatibility
100 | description: `${hit.snippet || hit.title} — Product: ${hit.product || hit.productId || "Unknown"} (${hit.version || hit.versionId || "Latest"})`,
101 | totalSnippets: 1,
102 | source: "help"
103 | }));
104 |
105 | // Store the full search results in a simple cache for retrieval
106 | // In a real implementation, you might want a more sophisticated cache
107 | if (!global.sapHelpSearchCache) {
108 | global.sapHelpSearchCache = new Map();
109 | }
110 | results.forEach(hit => {
111 | global.sapHelpSearchCache!.set(hit.loio, hit);
112 | });
113 |
114 | // Format response similar to other search functions
115 | const formattedResults = searchResults.slice(0, 20).map((result, i) =>
116 | `[${i}] **${result.title}**\n ID: \`${result.id}\`\n URL: ${result.url}\n ${result.description}\n`
117 | ).join('\n');
118 |
119 | return {
120 | results: searchResults.length > 0 ? searchResults : [{
121 | library_id: "sap-help",
122 | topic: '',
123 | id: "search-results",
124 | title: `SAP Help Search Results for "${query}"`,
125 | url: '',
126 | snippet: `Found ${searchResults.length} results from SAP Help:\n\n${formattedResults}\n\nUse sap_help_get with the ID of any result to retrieve the full content.`,
127 | score: 0,
128 | metadata: {
129 | source: "help",
130 | totalSnippets: searchResults.length
131 | },
132 | // Legacy fields for backward compatibility
133 | description: `Found ${searchResults.length} results from SAP Help:\n\n${formattedResults}\n\nUse sap_help_get with the ID of any result to retrieve the full content.`,
134 | totalSnippets: searchResults.length,
135 | source: "help"
136 | }]
137 | };
138 |
139 | } catch (error: any) {
140 | return {
141 | results: [],
142 | error: `SAP Help search error: ${error.message}`
143 | };
144 | }
145 | }
146 |
147 | /**
148 | * Get full content of a SAP Help page using the private APIs
149 | * First gets metadata, then page content
150 | */
151 | export async function getSapHelpContent(resultId: string): Promise<string> {
152 | try {
153 | // Extract loio from the result ID
154 | const loio = resultId.replace('sap-help-', '');
155 | if (!loio || loio === resultId) {
156 | throw new Error("Invalid SAP Help result ID. Use an ID from sap_help_search results.");
157 | }
158 |
159 | // First try to get from cache
160 | const cache = global.sapHelpSearchCache || new Map();
161 | let hit = cache.get(loio);
162 |
163 | if (!hit) {
164 | // If not in cache, search again to get the full hit data
165 | const searchParams = {
166 | transtype: "standard,html,pdf,others",
167 | state: "PRODUCTION,TEST,DRAFT",
168 | product: "",
169 | version: "",
170 | q: loio, // Search by loio to find the specific document
171 | to: "19",
172 | area: "content",
173 | advancedSearch: "0",
174 | excludeNotSearchable: "1",
175 | language: "en-US",
176 | };
177 |
178 | const searchUrl = `${BASE}/http.svc/elasticsearch?${toQuery(searchParams)}`;
179 | const searchResponse = await fetch(searchUrl, {
180 | headers: {
181 | Accept: "application/json",
182 | "User-Agent": "mcp-sap-docs/help-get",
183 | Referer: BASE,
184 | },
185 | });
186 |
187 | if (!searchResponse.ok) {
188 | throw new Error(`Failed to find document: ${searchResponse.status} ${searchResponse.statusText}`);
189 | }
190 |
191 | const searchData: SapHelpSearchResponse = await searchResponse.json();
192 | const results = searchData?.data?.results || [];
193 | hit = results.find(r => r.loio === loio);
194 |
195 | if (!hit) {
196 | throw new Error(`Document with loio ${loio} not found`);
197 | }
198 | }
199 |
200 | // Prepare metadata request parameters
201 | const topic_url = `${hit.loio}.html`;
202 | let product_url = hit.productId;
203 | let deliverable_url;
204 |
205 | try {
206 | const { productUrlSeg, deliverableLoio } = parseDocsPathParts(hit.url);
207 | deliverable_url = deliverableLoio;
208 | if (!product_url) product_url = productUrlSeg;
209 | } catch (e) {
210 | if (!product_url) {
211 | throw new Error("Could not determine product_url from hit; missing productId and unparsable url");
212 | }
213 | }
214 |
215 | const language = hit.language || "en-US";
216 |
217 | // Get deliverable metadata
218 | const metadataParams = {
219 | product_url,
220 | topic_url,
221 | version: "LATEST",
222 | loadlandingpageontopicnotfound: "true",
223 | deliverable_url,
224 | language,
225 | deliverableInfo: "1",
226 | toc: "1",
227 | };
228 |
229 | const metadataUrl = `${BASE}/http.svc/deliverableMetadata?${toQuery(metadataParams)}`;
230 | const metadataResponse = await fetch(metadataUrl, {
231 | headers: {
232 | Accept: "application/json",
233 | "User-Agent": "mcp-sap-docs/help-metadata",
234 | Referer: BASE,
235 | },
236 | });
237 |
238 | if (!metadataResponse.ok) {
239 | throw new Error(`Metadata request failed: ${metadataResponse.status} ${metadataResponse.statusText}`);
240 | }
241 |
242 | const metadataData: SapHelpMetadataResponse = await metadataResponse.json();
243 | const deliverable_id = metadataData?.data?.deliverable?.id;
244 | const buildNo = metadataData?.data?.deliverable?.buildNo;
245 | const file_path = metadataData?.data?.filePath || topic_url;
246 |
247 | if (!deliverable_id || !buildNo || !file_path) {
248 | throw new Error("Missing required metadata: deliverable_id, buildNo, or file_path");
249 | }
250 |
251 | // Get page content
252 | const pageParams = {
253 | deliverableInfo: "1",
254 | deliverable_id,
255 | buildNo,
256 | file_path,
257 | };
258 |
259 | const pageUrl = `${BASE}/http.svc/pagecontent?${toQuery(pageParams)}`;
260 | const pageResponse = await fetch(pageUrl, {
261 | headers: {
262 | Accept: "application/json",
263 | "User-Agent": "mcp-sap-docs/help-content",
264 | Referer: BASE,
265 | },
266 | });
267 |
268 | if (!pageResponse.ok) {
269 | throw new Error(`Page content request failed: ${pageResponse.status} ${pageResponse.statusText}`);
270 | }
271 |
272 | const pageData: SapHelpPageContentResponse = await pageResponse.json();
273 | const title = pageData?.data?.currentPage?.t || pageData?.data?.deliverable?.title || hit.title;
274 | const bodyHtml = pageData?.data?.body || "";
275 |
276 | if (!bodyHtml) {
277 | return `# ${title}\n\nNo content available for this page.`;
278 | }
279 |
280 | // Convert HTML to readable text while preserving structure
281 | const cleanText = bodyHtml
282 | .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '') // Remove scripts
283 | .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '') // Remove styles
284 | .replace(/<h([1-6])[^>]*>/gi, (_, level) => '\n' + '#'.repeat(parseInt(level)) + ' ') // Convert headings
285 | .replace(/<\/h[1-6]>/gi, '\n') // Close headings
286 | .replace(/<p[^>]*>/gi, '\n') // Paragraphs
287 | .replace(/<\/p>/gi, '\n')
288 | .replace(/<br[^>]*>/gi, '\n') // Line breaks
289 | .replace(/<li[^>]*>/gi, '• ') // List items
290 | .replace(/<\/li>/gi, '\n')
291 | .replace(/<code[^>]*>/gi, '`') // Inline code
292 | .replace(/<\/code>/gi, '`')
293 | .replace(/<pre[^>]*>/gi, '\n```\n') // Code blocks
294 | .replace(/<\/pre>/gi, '\n```\n')
295 | .replace(/<[^>]+>/g, '') // Remove remaining HTML tags
296 | .replace(/\s*\n\s*\n\s*/g, '\n\n') // Clean up multiple newlines
297 | .replace(/^\s+|\s+$/g, '') // Trim
298 | .trim();
299 |
300 | // Build the full content with metadata
301 | const fullContent = `# ${title}
302 |
303 | **Source:** SAP Help Portal
304 | **URL:** ${ensureAbsoluteUrl(hit.url)}
305 | **Product:** ${hit.product || hit.productId || "Unknown"}
306 | **Version:** ${hit.version || hit.versionId || "Latest"}
307 | **Language:** ${hit.language || "en-US"}
308 | ${hit.snippet ? `**Summary:** ${hit.snippet}` : ''}
309 |
310 | ---
311 |
312 | ${cleanText}
313 |
314 | ---
315 |
316 | *This content is from the SAP Help Portal and represents official SAP documentation.*`;
317 |
318 | // Apply intelligent truncation if content is too large
319 | const truncationResult = truncateContent(fullContent);
320 |
321 | return truncationResult.content;
322 |
323 | } catch (error: any) {
324 | throw new Error(`Failed to get SAP Help content: ${error.message}`);
325 | }
326 | }
```
--------------------------------------------------------------------------------
/index.html:
--------------------------------------------------------------------------------
```html
1 | <!DOCTYPE html>
2 | <html lang="en">
3 | <head>
4 | <meta charset="UTF-8">
5 | <meta name="viewport" content="width=device-width, initial-scale=1.0">
6 | <title>SAP Documentation MCP Server</title>
7 | <style>
8 | body {
9 | font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen', 'Ubuntu', 'Cantarell', sans-serif;
10 | line-height: 1.6;
11 | color: #24292e;
12 | max-width: 1200px;
13 | margin: 0 auto;
14 | padding: 20px;
15 | background-color: #f6f8fa;
16 | }
17 | .container {
18 | background-color: white;
19 | padding: 40px;
20 | border-radius: 8px;
21 | box-shadow: 0 1px 3px rgba(0,0,0,0.12), 0 1px 2px rgba(0,0,0,0.24);
22 | }
23 | h1 {
24 | color: #0366d6;
25 | border-bottom: 2px solid #e1e4e8;
26 | padding-bottom: 10px;
27 | }
28 | h2 {
29 | color: #24292e;
30 | margin-top: 30px;
31 | border-bottom: 1px solid #e1e4e8;
32 | padding-bottom: 5px;
33 | }
34 | .highlight {
35 | background-color: #f1f8ff;
36 | border: 1px solid #c8e1ff;
37 | border-radius: 6px;
38 | padding: 16px;
39 | margin: 16px 0;
40 | }
41 | .server-url {
42 | font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace;
43 | background-color: #f6f8fa;
44 | padding: 2px 4px;
45 | border-radius: 3px;
46 | font-size: 14px;
47 | }
48 | .feature-grid {
49 | display: grid;
50 | grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
51 | gap: 20px;
52 | margin: 20px 0;
53 | }
54 | .feature-card {
55 | background-color: #f6f8fa;
56 | padding: 20px;
57 | border-radius: 6px;
58 | border-left: 4px solid #0366d6;
59 | }
60 | .feature-card h3 {
61 | margin-top: 0;
62 | color: #0366d6;
63 | }
64 | code {
65 | background-color: rgba(27,31,35,0.05);
66 | padding: 2px 4px;
67 | border-radius: 3px;
68 | font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace;
69 | font-size: 14px;
70 | }
71 | .status-links {
72 | display: flex;
73 | gap: 10px;
74 | flex-wrap: wrap;
75 | margin: 20px 0;
76 | }
77 | .status-link {
78 | background-color: #0366d6;
79 | color: white;
80 | padding: 8px 16px;
81 | text-decoration: none;
82 | border-radius: 6px;
83 | font-size: 14px;
84 | transition: background-color 0.2s;
85 | }
86 | .status-link:hover {
87 | background-color: #0256cc;
88 | }
89 | .coverage-stats {
90 | display: grid;
91 | grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
92 | gap: 15px;
93 | margin: 20px 0;
94 | }
95 | .stat-card {
96 | text-align: center;
97 | padding: 15px;
98 | background-color: #f1f8ff;
99 | border-radius: 6px;
100 | }
101 | .stat-number {
102 | font-size: 24px;
103 | font-weight: bold;
104 | color: #0366d6;
105 | display: block;
106 | }
107 | .tools-list {
108 | list-style: none;
109 | padding: 0;
110 | display: grid;
111 | grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
112 | gap: 10px;
113 | }
114 | .tools-list li {
115 | background-color: #f6f8fa;
116 | padding: 10px 15px;
117 | border-radius: 6px;
118 | border-left: 3px solid #28a745;
119 | }
120 | .example-section {
121 | background-color: #f8f9fa;
122 | border-radius: 6px;
123 | padding: 20px;
124 | margin: 20px 0;
125 | }
126 | .example-section h3 {
127 | margin-top: 0;
128 | color: #24292e;
129 | }
130 | .example-queries {
131 | display: grid;
132 | grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
133 | gap: 15px;
134 | }
135 | .query-card {
136 | background-color: white;
137 | padding: 15px;
138 | border-radius: 6px;
139 | border: 1px solid #e1e4e8;
140 | }
141 | .query-card strong {
142 | color: #0366d6;
143 | }
144 | </style>
145 | </head>
146 | <body>
147 | <div class="container">
148 | <h1>🚀 SAP Documentation MCP Server</h1>
149 |
150 | <div class="highlight">
151 | <strong>Welcome!</strong> This server provides unified access to official SAP documentation, community content, and help portal resources through the Model Context Protocol (MCP).
152 | </div>
153 |
154 | <div class="status-links">
155 | <a href="/status" class="status-link">📊 Server Status</a>
156 | <a href="/healthz" class="status-link">🏥 Health Check</a>
157 | <a href="https://mcp-sap-docs.marianzeis.de/mcp" class="status-link">🌐 Public MCP Endpoint</a>
158 | </div>
159 |
160 | <h2>🔗 Quick Start</h2>
161 | <p>Connect your MCP client to this server using one of these methods:</p>
162 |
163 | <div class="feature-grid">
164 | <div class="feature-card">
165 | <h3>🌐 Remote Connection (Recommended)</h3>
166 | <p>Use the public MCP Streamable HTTP endpoint:</p>
167 | <div class="server-url">https://mcp-sap-docs.marianzeis.de/mcp</div>
168 | <p>Perfect for Claude Desktop, VS Code, Cursor, and other MCP clients.</p>
169 | </div>
170 |
171 | <div class="feature-card">
172 | <h3>💻 Local STDIO</h3>
173 | <p>Run locally with:</p>
174 | <code>node dist/src/server.js</code>
175 | <p>For development and local testing.</p>
176 | </div>
177 |
178 | <div class="feature-card">
179 | <h3>🔄 Streamable HTTP</h3>
180 | <p>Latest MCP protocol support:</p>
181 | <code>http://127.0.0.1:3122/mcp</code>
182 | <p>Enhanced session management and resumability.</p>
183 | </div>
184 | </div>
185 |
186 | <h2>🛠️ Available Tools</h2>
187 | <ul class="tools-list">
188 | <li><strong>sap_docs_search</strong> - Unified search across SAPUI5/CAP/OpenUI5 APIs & samples, wdi5, and more</li>
189 | <li><strong>sap_community_search</strong> - Real-time SAP Community posts with full content of top 3 results</li>
190 | <li><strong>sap_help_search</strong> - Comprehensive search across all SAP Help Portal documentation</li>
191 | <li><strong>sap_docs_get</strong> - Fetches full documents/snippets with smart formatting</li>
192 | <li><strong>sap_help_get</strong> - Retrieves complete SAP Help pages with metadata</li>
193 | </ul>
194 |
195 | <h2>📚 Documentation Coverage</h2>
196 | <div class="coverage-stats">
197 | <div class="stat-card">
198 | <span class="stat-number">1,485+</span>
199 | SAPUI5 Documentation Files
200 | </div>
201 | <div class="stat-card">
202 | <span class="stat-number">195+</span>
203 | CAP Documentation Files
204 | </div>
205 | <div class="stat-card">
206 | <span class="stat-number">500+</span>
207 | OpenUI5 API Controls
208 | </div>
209 | <div class="stat-card">
210 | <span class="stat-number">2,000+</span>
211 | Sample Code Files
212 | </div>
213 | <div class="stat-card">
214 | <span class="stat-number">Real-time</span>
215 | Community Content
216 | </div>
217 | <div class="stat-card">
218 | <span class="stat-number">Complete</span>
219 | SAP Help Portal
220 | </div>
221 | </div>
222 |
223 | <div class="example-section">
224 | <h3>💡 Example Queries</h3>
225 | <div class="example-queries">
226 | <div class="query-card">
227 | <strong>Official Documentation:</strong><br>
228 | "How do I implement authentication in SAPUI5?"<br>
229 | "Show me wdi5 testing examples for forms"<br>
230 | "Find OpenUI5 button control examples"
231 | </div>
232 | <div class="query-card">
233 | <strong>Community Knowledge:</strong><br>
234 | "Latest CAP authentication best practices from community"<br>
235 | "Community examples of OData batch operations"<br>
236 | "Temporal data handling in CAP solutions"
237 | </div>
238 | <div class="query-card">
239 | <strong>SAP Help Portal:</strong><br>
240 | "How to configure S/4HANA Fiori Launchpad?"<br>
241 | "BTP integration documentation for Analytics Cloud"<br>
242 | "ABAP development best practices in S/4HANA"
243 | </div>
244 | </div>
245 | </div>
246 |
247 | <h2>🏛️ Architecture</h2>
248 | <div class="feature-grid">
249 | <div class="feature-card">
250 | <h3>🧠 MCP Server</h3>
251 | <p>Node.js/TypeScript server exposing SAP documentation resources and tools</p>
252 | </div>
253 | <div class="feature-card">
254 | <h3>🔄 Streamable HTTP</h3>
255 | <p>Latest MCP spec (2025-06-18) with session management and resumability</p>
256 | </div>
257 | <div class="feature-card">
258 | <h3>🔍 Search Engine</h3>
259 | <p>SQLite FTS5 + JSON indices for fast local search</p>
260 | </div>
261 | <div class="feature-card">
262 | <h3>👥 Community Integration</h3>
263 | <p>HTML scraping + LiQL API for full content retrieval</p>
264 | </div>
265 | <div class="feature-card">
266 | <h3>📖 SAP Help Integration</h3>
267 | <p>Private API access to help.sap.com content</p>
268 | </div>
269 | </div>
270 |
271 | <h2>🔧 For Developers</h2>
272 | <div class="highlight">
273 | <h3>Build Commands:</h3>
274 | <code>npm run build</code> - Compile TypeScript<br>
275 | <code>npm run build:index</code> - Build search index<br>
276 | <code>npm run build:fts</code> - Build FTS5 database<br><br>
277 |
278 | <h3>Server Commands:</h3>
279 | <code>npm start</code> - Start STDIO MCP server<br>
280 | <code>npm run start:http</code> - Start HTTP status server (port 3001)<br>
281 | <code>npm run start:streamable</code> - Start Streamable HTTP MCP server (port 3122)<br>
282 | </div>
283 |
284 | <div style="margin-top: 40px; padding-top: 20px; border-top: 1px solid #e1e4e8; text-align: center; color: #6a737d;">
285 | <p>📊 <strong>Total Coverage:</strong> 4,180+ documentation files + real-time community & help portal content</p>
286 | <p>🔗 <a href="https://github.com/marianfoo/mcp-sap-docs" style="color: #0366d6;">View on GitHub</a> |
287 | 📄 <a href="https://github.com/marianfoo/mcp-sap-docs/blob/main/README.md" style="color: #0366d6;">Full Documentation</a></p>
288 | </div>
289 | </div>
290 | </body>
291 | </html>
```
--------------------------------------------------------------------------------
/test/tools/run-tests.js:
--------------------------------------------------------------------------------
```javascript
1 | // Unified test runner for MCP SAP Docs - supports both all tests and specific files
2 | import { readdirSync } from 'node:fs';
3 | import { join, dirname } from 'node:path';
4 | import { fileURLToPath } from 'node:url';
5 | import { startServerHttp, waitForStatus, stopServer, docsSearch } from '../_utils/httpClient.js';
6 |
7 | // ANSI color codes
8 | const colors = {
9 | reset: '\x1b[0m',
10 | bright: '\x1b[1m',
11 | dim: '\x1b[2m',
12 | red: '\x1b[31m',
13 | green: '\x1b[32m',
14 | yellow: '\x1b[33m',
15 | blue: '\x1b[34m',
16 | magenta: '\x1b[35m',
17 | cyan: '\x1b[36m',
18 | white: '\x1b[37m'
19 | };
20 |
21 | function colorize(text, color) {
22 | return `${colors[color]}${text}${colors.reset}`;
23 | }
24 |
25 | const __filename = fileURLToPath(import.meta.url);
26 | const ROOT = dirname(__filename);
27 | const TOOLS_DIR = join(ROOT);
28 |
29 | function listJsFiles(dir) {
30 | const entries = readdirSync(dir, { withFileTypes: true });
31 | const files = [];
32 | for (const e of entries) {
33 | const p = join(dir, e.name);
34 | if (e.isDirectory()) files.push(...listJsFiles(p));
35 | else if (e.isFile() && e.name.endsWith('.js')) files.push(p);
36 | }
37 | return files;
38 | }
39 |
40 | function parseArgs() {
41 | const args = process.argv.slice(2);
42 | const config = {
43 | specificFile: null,
44 | showHelp: false
45 | };
46 |
47 | for (let i = 0; i < args.length; i++) {
48 | const arg = args[i];
49 | if (arg === '--spec' && i + 1 < args.length) {
50 | config.specificFile = args[i + 1];
51 | i++; // Skip next argument since it's the file path
52 | } else if (arg === '--help' || arg === '-h') {
53 | config.showHelp = true;
54 | }
55 | }
56 |
57 | return config;
58 | }
59 |
60 | function showHelp() {
61 | console.log(colorize('MCP SAP Docs Test Runner', 'cyan'));
62 | console.log('');
63 | console.log(colorize('Usage:', 'yellow'));
64 | console.log(' npm run test Run all test files');
65 | console.log(' npm run test -- --spec <file-path> Run specific test file');
66 | console.log('');
67 | console.log(colorize('Examples:', 'yellow'));
68 | console.log(' npm run test');
69 | console.log(' npm run test:fast Skip build step');
70 | console.log(' npm run test -- --spec search-cap-docs.js');
71 | console.log(' npm run test:smoke Quick smoke test');
72 | console.log('');
73 | console.log(colorize('Available test files:', 'yellow'));
74 |
75 | const allFiles = listJsFiles(TOOLS_DIR)
76 | .filter(p => !p.endsWith('run-all.js') && !p.endsWith('run-single.js') && !p.endsWith('run-tests.js'));
77 |
78 | allFiles.forEach(f => {
79 | // Show relative path from project root
80 | const relativePath = f.replace(process.cwd() + '/', '');
81 | console.log(colorize(` ${relativePath}`, 'cyan'));
82 | });
83 | }
84 |
85 | function findTestFile(pattern) {
86 | const allFiles = listJsFiles(TOOLS_DIR)
87 | .filter(p => !p.endsWith('run-all.js') && !p.endsWith('run-single.js') && !p.endsWith('run-tests.js'));
88 |
89 | // Try different matching strategies
90 | let matches = [];
91 |
92 | // 1. Exact path match (relative to project root or absolute)
93 | if (pattern.startsWith('/')) {
94 | matches = allFiles.filter(f => f === pattern);
95 | } else {
96 | // Try as relative path from project root
97 | const fullPattern = join(process.cwd(), pattern);
98 | matches = allFiles.filter(f => f === fullPattern);
99 | }
100 |
101 | // 2. If no exact match, try partial path matching
102 | if (matches.length === 0) {
103 | matches = allFiles.filter(f => f.includes(pattern));
104 | }
105 |
106 | // 3. If still no match, try just filename matching
107 | if (matches.length === 0) {
108 | matches = allFiles.filter(f => f.split('/').pop() === pattern);
109 | }
110 |
111 | if (matches.length === 0) {
112 | console.log(colorize(`❌ No test file found matching: ${pattern}`, 'red'));
113 | console.log(colorize('Available test files:', 'yellow'));
114 | allFiles.forEach(f => {
115 | const relativePath = f.replace(process.cwd() + '/', '');
116 | console.log(colorize(` ${relativePath}`, 'cyan'));
117 | });
118 | process.exit(1);
119 | }
120 |
121 | if (matches.length > 1) {
122 | console.log(colorize(`⚠️ Multiple files match "${pattern}":`, 'yellow'));
123 | matches.forEach(f => {
124 | const relativePath = f.replace(process.cwd() + '/', '');
125 | console.log(colorize(` ${relativePath}`, 'cyan'));
126 | });
127 | console.log(colorize('Please be more specific.', 'yellow'));
128 | process.exit(1);
129 | }
130 |
131 | return matches[0];
132 | }
133 |
134 | async function runTestFile(filePath, fileName) {
135 | console.log(colorize(`📁 Running ${fileName}`, 'blue'));
136 | console.log(colorize('─'.repeat(50), 'dim'));
137 |
138 | // Load and run the test file
139 | const mod = await import(fileURLToPath(new URL(filePath, import.meta.url)));
140 | const cases = (mod.default || []).flat();
141 |
142 | if (cases.length === 0) {
143 | console.log(colorize('⚠️ No test cases found in file', 'yellow'));
144 | return { tests: 0, failures: 0 };
145 | }
146 |
147 | let fileFailures = 0;
148 | let fileTests = 0;
149 |
150 | for (const c of cases) {
151 | try {
152 | if (typeof c.validate === 'function') {
153 | // New path: custom validator gets helpers, uses existing server
154 | const res = await c.validate({ docsSearch });
155 |
156 | if (res && typeof res === 'object' && res.skipped) {
157 | const reason = res.message ? ` - ${res.message}` : '';
158 | console.log(` ${colorize('⚠️', 'yellow')} ${colorize(c.name, 'white')} (skipped${reason})`);
159 | continue;
160 | }
161 |
162 | fileTests++; // Only count tests that are actually executed
163 | const passed = typeof res === 'object' ? !!res.passed : !!res;
164 | if (!passed) {
165 | const msg = (res && res.message) ? ` - ${res.message}` : '';
166 | throw new Error(`custom validator failed${msg}`);
167 | }
168 | console.log(` ${colorize('✅', 'green')} ${colorize(c.name, 'white')}`);
169 | } else {
170 | // Legacy path: expectIncludes (kept for existing tests)
171 | const text = await docsSearch(c.query);
172 |
173 | if (c.skipIfNoResults && /No results found for/.test(text)) {
174 | console.log(` ${colorize('⚠️', 'yellow')} ${colorize(c.name, 'white')} (skipped - no results available)`);
175 | continue;
176 | }
177 |
178 | fileTests++; // Only count tests that are actually executed
179 |
180 | // Check expectIncludes
181 | if (c.expectIncludes) {
182 | const checks = Array.isArray(c.expectIncludes) ? c.expectIncludes : [c.expectIncludes];
183 | const ok = checks.every(expectedFragment => {
184 | // Direct match (exact inclusion)
185 | if (text.includes(expectedFragment)) {
186 | return true;
187 | }
188 |
189 | // If expected fragment is a parent document (no #), check if any section from that document is found
190 | if (!expectedFragment.includes('#')) {
191 | // Look for any section that starts with the expected parent document path followed by #
192 | const sectionPattern = new RegExp(expectedFragment.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '#[^\\s]*', 'g');
193 | return sectionPattern.test(text);
194 | }
195 |
196 | return false;
197 | });
198 | if (!ok) throw new Error(`expected fragment(s) not found: ${checks.join(', ')}`);
199 | }
200 |
201 | // Check expectContains (for URL verification)
202 | if (c.expectContains) {
203 | const containsChecks = Array.isArray(c.expectContains) ? c.expectContains : [c.expectContains];
204 | const containsOk = containsChecks.every(expectedContent => {
205 | return text.includes(expectedContent);
206 | });
207 | if (!containsOk) throw new Error(`expected content not found: ${containsChecks.join(', ')}`);
208 | }
209 |
210 | // Check expectUrlPattern (for URL format verification)
211 | if (c.expectUrlPattern) {
212 | // Extract URLs from the response using the 🔗 emoji
213 | const urlRegex = /🔗\s+(https?:\/\/[^\s\n]+)/g;
214 | const urls = [];
215 | let match;
216 | while ((match = urlRegex.exec(text)) !== null) {
217 | urls.push(match[1]);
218 | }
219 |
220 | if (urls.length === 0) {
221 | throw new Error('no URLs found in response (expected URL pattern)');
222 | }
223 |
224 | const urlPattern = c.expectUrlPattern;
225 | const matchingUrl = urls.some(url => {
226 | if (typeof urlPattern === 'string') {
227 | return url.includes(urlPattern) || new RegExp(urlPattern).test(url);
228 | }
229 | return urlPattern.test(url);
230 | });
231 |
232 | if (!matchingUrl) {
233 | throw new Error(`no URL matching pattern "${urlPattern}" found. URLs found: ${urls.join(', ')}`);
234 | }
235 | }
236 |
237 | // Check expectPattern (for general regex pattern matching)
238 | if (c.expectPattern) {
239 | const pattern = c.expectPattern;
240 | if (!pattern.test(text)) {
241 | throw new Error(`text does not match expected pattern: ${pattern}`);
242 | }
243 | }
244 |
245 | console.log(` ${colorize('✅', 'green')} ${colorize(c.name, 'white')}`);
246 | }
247 | } catch (err) {
248 | fileFailures++;
249 | console.log(` ${colorize('❌', 'red')} ${colorize(c.name, 'white')}: ${colorize(err?.message || err, 'red')}`);
250 | }
251 | }
252 |
253 | return { tests: fileTests, failures: fileFailures };
254 | }
255 |
256 |
257 |
258 | async function runTests() {
259 | const config = parseArgs();
260 |
261 | if (config.showHelp) {
262 | showHelp();
263 | process.exit(0);
264 | }
265 |
266 |
267 |
268 | let testFiles = [];
269 |
270 | if (config.specificFile) {
271 | // Run specific test file
272 | const testFile = findTestFile(config.specificFile);
273 | const fileName = testFile.split('/').pop();
274 | console.log(colorize(`🚀 Running specific test: ${fileName}`, 'cyan'));
275 | testFiles = [testFile];
276 | } else {
277 | // Run all test files
278 | console.log(colorize('🚀 Starting MCP SAP Docs test suite...', 'cyan'));
279 | testFiles = listJsFiles(TOOLS_DIR)
280 | .filter(p => {
281 | const fileName = p.split('/').pop();
282 | // Skip runner scripts and utility files
283 | return !fileName.startsWith('run-') &&
284 | !fileName.includes('test-with-reranker') &&
285 | fileName.endsWith('.js');
286 | })
287 | .sort();
288 | }
289 |
290 | // Start HTTP server
291 | const server = startServerHttp();
292 | let totalFailures = 0;
293 | let totalTests = 0;
294 |
295 | try {
296 | console.log(colorize('⏳ Waiting for server to be ready...', 'yellow'));
297 | await waitForStatus();
298 | console.log(colorize('✅ Server ready!\n', 'green'));
299 |
300 | for (const file of testFiles) {
301 | const fileName = file.split('/').pop();
302 |
303 | // Add spacing between files when running multiple
304 | if (testFiles.length > 1) {
305 | console.log('');
306 | }
307 |
308 | const result = await runTestFile(file, fileName);
309 | totalTests += result.tests;
310 | totalFailures += result.failures;
311 | }
312 | } finally {
313 | await stopServer(server);
314 | }
315 |
316 | console.log(colorize('\n' + '═'.repeat(60), 'dim'));
317 |
318 | if (totalFailures) {
319 | console.log(`${colorize('❌ Test Results:', 'red')} ${colorize(`${totalFailures}/${totalTests} tests failed`, 'red')}`);
320 | process.exit(1);
321 | } else {
322 | console.log(`${colorize('🎉 Test Results:', 'green')} ${colorize(`All ${totalTests} tests passed!`, 'green')}`);
323 | }
324 | }
325 |
326 | runTests().catch(err => {
327 | console.error(colorize('Fatal error:', 'red'), err);
328 | process.exit(1);
329 | });
330 |
```
--------------------------------------------------------------------------------
/test/validate-urls.ts:
--------------------------------------------------------------------------------
```typescript
1 | #!/usr/bin/env node
2 | /**
3 | * URL Validation Script
4 | * Tests random URLs from each documentation source to verify they're not 404
5 | */
6 |
7 | import { fileURLToPath } from 'url';
8 | import { dirname, join } from 'path';
9 | import fs from 'fs/promises';
10 | import { existsSync } from 'fs';
11 | import { generateDocumentationUrl } from '../src/lib/url-generation/index.js';
12 | import { getDocUrlConfig, getSourcePath } from '../src/lib/metadata.js';
13 |
14 | const __filename = fileURLToPath(import.meta.url);
15 | const __dirname = dirname(__filename);
16 | const PROJECT_ROOT = join(__dirname, '..');
17 | const DATA_DIR = join(PROJECT_ROOT, 'dist', 'data');
18 |
19 | interface TestResult {
20 | source: string;
21 | url: string;
22 | status: number;
23 | ok: boolean;
24 | error?: string;
25 | docTitle: string;
26 | relFile: string;
27 | responseTime: number;
28 | }
29 |
30 | interface LibraryBundle {
31 | id: string;
32 | name: string;
33 | description: string;
34 | docs: {
35 | id: string;
36 | title: string;
37 | description: string;
38 | snippetCount: number;
39 | relFile: string;
40 | }[];
41 | }
42 |
43 | // Colors for console output
44 | const colors = {
45 | reset: '\x1b[0m',
46 | red: '\x1b[31m',
47 | green: '\x1b[32m',
48 | yellow: '\x1b[33m',
49 | blue: '\x1b[34m',
50 | magenta: '\x1b[35m',
51 | cyan: '\x1b[36m',
52 | bold: '\x1b[1m',
53 | dim: '\x1b[2m'
54 | };
55 |
56 | async function loadIndex(): Promise<Record<string, LibraryBundle>> {
57 | const indexPath = join(DATA_DIR, 'index.json');
58 | if (!existsSync(indexPath)) {
59 | throw new Error(`Index file not found: ${indexPath}. Run 'npm run build' first.`);
60 | }
61 |
62 | const raw = await fs.readFile(indexPath, 'utf8');
63 | return JSON.parse(raw) as Record<string, LibraryBundle>;
64 | }
65 |
66 | function getRandomItems<T>(array: T[], count: number): T[] {
67 | const shuffled = [...array].sort(() => 0.5 - Math.random());
68 | return shuffled.slice(0, Math.min(count, array.length));
69 | }
70 |
71 | async function getDocumentContent(libraryId: string, relFile: string): Promise<string> {
72 | const sourcePath = getSourcePath(libraryId);
73 | if (!sourcePath) {
74 | throw new Error(`Unknown library ID: ${libraryId}`);
75 | }
76 |
77 | const fullPath = join(PROJECT_ROOT, 'sources', sourcePath, relFile);
78 | if (!existsSync(fullPath)) {
79 | return '# No content available';
80 | }
81 |
82 | return await fs.readFile(fullPath, 'utf8');
83 | }
84 |
85 | async function testUrl(url: string, timeout: number = 10000): Promise<{ status: number; ok: boolean; error?: string; responseTime: number }> {
86 | const startTime = Date.now();
87 |
88 | try {
89 | const controller = new AbortController();
90 | const timeoutId = setTimeout(() => controller.abort(), timeout);
91 |
92 | const response = await fetch(url, {
93 | method: 'HEAD', // Use HEAD to avoid downloading full content
94 | signal: controller.signal,
95 | headers: {
96 | 'User-Agent': 'SAP-Docs-MCP-URL-Validator/1.0'
97 | }
98 | });
99 |
100 | clearTimeout(timeoutId);
101 | const responseTime = Date.now() - startTime;
102 |
103 | return {
104 | status: response.status,
105 | ok: response.ok,
106 | responseTime
107 | };
108 | } catch (error: any) {
109 | const responseTime = Date.now() - startTime;
110 |
111 | if (error.name === 'AbortError') {
112 | return {
113 | status: 0,
114 | ok: false,
115 | error: 'Timeout',
116 | responseTime
117 | };
118 | }
119 |
120 | return {
121 | status: 0,
122 | ok: false,
123 | error: error.message || 'Network error',
124 | responseTime
125 | };
126 | }
127 | }
128 |
129 | async function generateUrlForDoc(libraryId: string, doc: any): Promise<string | null> {
130 | const config = getDocUrlConfig(libraryId);
131 | if (!config) {
132 | console.warn(`${colors.yellow}⚠️ No URL config for ${libraryId}${colors.reset}`);
133 | return null;
134 | }
135 |
136 | try {
137 | const content = await getDocumentContent(libraryId, doc.relFile);
138 | return generateDocumentationUrl(libraryId, doc.relFile, content, config);
139 | } catch (error) {
140 | console.warn(`${colors.yellow}⚠️ Could not read content for ${doc.relFile}: ${error}${colors.reset}`);
141 | return null;
142 | }
143 | }
144 |
145 | async function validateSourceUrls(library: LibraryBundle, sampleSize: number = 5): Promise<TestResult[]> {
146 | console.log(`\n${colors.cyan}📚 Testing ${library.name} (${library.id})${colors.reset}`);
147 | console.log(`${colors.dim} ${library.description}${colors.reset}`);
148 |
149 | // Get random sample of documents
150 | const randomDocs = getRandomItems(library.docs, sampleSize);
151 | console.log(`${colors.blue} Selected ${randomDocs.length} random documents${colors.reset}`);
152 |
153 | const results: TestResult[] = [];
154 | const promises = randomDocs.map(async (doc) => {
155 | const url = await generateUrlForDoc(library.id, doc);
156 |
157 | if (!url) {
158 | return {
159 | source: library.id,
160 | url: 'N/A',
161 | status: 0,
162 | ok: false,
163 | error: 'Could not generate URL',
164 | docTitle: doc.title,
165 | relFile: doc.relFile,
166 | responseTime: 0
167 | };
168 | }
169 |
170 | console.log(`${colors.dim} Testing: ${url}${colors.reset}`);
171 | const testResult = await testUrl(url);
172 |
173 | return {
174 | source: library.id,
175 | url,
176 | status: testResult.status,
177 | ok: testResult.ok,
178 | error: testResult.error,
179 | docTitle: doc.title,
180 | relFile: doc.relFile,
181 | responseTime: testResult.responseTime
182 | };
183 | });
184 |
185 | // Wait for all tests to complete
186 | const testResults = await Promise.all(promises);
187 | results.push(...testResults);
188 |
189 | // Display results for this source
190 | const successful = results.filter(r => r.ok).length;
191 | const failed = results.filter(r => !r.ok).length;
192 |
193 | console.log(`${colors.bold} Results: ${colors.green}✅ ${successful} OK${colors.reset}${colors.bold}, ${colors.red}❌ ${failed} Failed${colors.reset}`);
194 |
195 | // Show detailed results
196 | results.forEach(result => {
197 | const statusColor = result.ok ? colors.green : colors.red;
198 | const statusIcon = result.ok ? '✅' : '❌';
199 | const statusText = result.status > 0 ? result.status.toString() : (result.error || 'ERROR');
200 |
201 | console.log(` ${statusIcon} ${statusColor}[${statusText}]${colors.reset} ${result.docTitle}`);
202 | console.log(` ${colors.dim}${result.url}${colors.reset}`);
203 | if (!result.ok && result.error) {
204 | console.log(` ${colors.red}Error: ${result.error}${colors.reset}`);
205 | }
206 | if (result.responseTime > 0) {
207 | console.log(` ${colors.dim}Response time: ${result.responseTime}ms${colors.reset}`);
208 | }
209 | });
210 |
211 | return results;
212 | }
213 |
214 | async function generateSummaryReport(allResults: TestResult[]) {
215 | console.log(`\n${colors.bold}${colors.cyan}📊 SUMMARY REPORT${colors.reset}`);
216 | console.log(`${'='.repeat(60)}`);
217 |
218 | const totalTests = allResults.length;
219 | const successfulTests = allResults.filter(r => r.ok).length;
220 | const failedTests = allResults.filter(r => !r.ok).length;
221 | const successRate = totalTests > 0 ? ((successfulTests / totalTests) * 100).toFixed(1) : '0.0';
222 |
223 | console.log(`${colors.bold}Overall Results:${colors.reset}`);
224 | console.log(` Total URLs tested: ${colors.bold}${totalTests}${colors.reset}`);
225 | console.log(` Successful: ${colors.green}${successfulTests}${colors.reset}`);
226 | console.log(` Failed: ${colors.red}${failedTests}${colors.reset}`);
227 | console.log(` Success rate: ${colors.bold}${successRate}%${colors.reset}`);
228 |
229 | // Group by source
230 | const bySource = allResults.reduce((acc, result) => {
231 | if (!acc[result.source]) {
232 | acc[result.source] = { total: 0, successful: 0, failed: 0 };
233 | }
234 | acc[result.source].total++;
235 | if (result.ok) {
236 | acc[result.source].successful++;
237 | } else {
238 | acc[result.source].failed++;
239 | }
240 | return acc;
241 | }, {} as Record<string, { total: number; successful: number; failed: number }>);
242 |
243 | console.log(`\n${colors.bold}By Source:${colors.reset}`);
244 | Object.entries(bySource).forEach(([source, stats]) => {
245 | const rate = ((stats.successful / stats.total) * 100).toFixed(1);
246 | const rateColor = stats.successful === stats.total ? colors.green :
247 | stats.successful > stats.total / 2 ? colors.yellow : colors.red;
248 | console.log(` ${source}: ${rateColor}${rate}%${colors.reset} (${colors.green}${stats.successful}${colors.reset}/${stats.total})`);
249 | });
250 |
251 | // Show failed URLs
252 | const failed = allResults.filter(r => !r.ok);
253 | if (failed.length > 0) {
254 | console.log(`\n${colors.bold}${colors.red}❌ Failed URLs:${colors.reset}`);
255 | failed.forEach(result => {
256 | console.log(` ${colors.red}[${result.status || 'ERROR'}]${colors.reset} ${result.url}`);
257 | console.log(` ${colors.dim}${result.docTitle} (${result.source})${colors.reset}`);
258 | if (result.error) {
259 | console.log(` ${colors.red}${result.error}${colors.reset}`);
260 | }
261 | });
262 | }
263 |
264 | // Performance stats
265 | const responseTimes = allResults.filter(r => r.responseTime > 0).map(r => r.responseTime);
266 | if (responseTimes.length > 0) {
267 | const avgResponseTime = Math.round(responseTimes.reduce((a, b) => a + b, 0) / responseTimes.length);
268 | const maxResponseTime = Math.max(...responseTimes);
269 | console.log(`\n${colors.bold}Performance:${colors.reset}`);
270 | console.log(` Average response time: ${avgResponseTime}ms`);
271 | console.log(` Slowest response: ${maxResponseTime}ms`);
272 | }
273 | }
274 |
275 | async function main() {
276 | console.log(`${colors.bold}${colors.blue}🔗 SAP Docs MCP - URL Validation Tool${colors.reset}`);
277 | console.log(`Testing random URLs from each documentation source...\n`);
278 |
279 | try {
280 | const index = await loadIndex();
281 | const sources = Object.values(index);
282 |
283 | console.log(`${colors.bold}Found ${sources.length} documentation sources:${colors.reset}`);
284 | sources.forEach(lib => {
285 | const hasUrlConfig = getDocUrlConfig(lib.id) !== null;
286 | const configStatus = hasUrlConfig ? `${colors.green}✅${colors.reset}` : `${colors.red}❌${colors.reset}`;
287 | console.log(` ${configStatus} ${lib.name} (${lib.id}) - ${lib.docs.length} docs`);
288 | });
289 |
290 | const sourcesWithUrls = sources.filter(lib => getDocUrlConfig(lib.id) !== null);
291 |
292 | if (sourcesWithUrls.length === 0) {
293 | console.log(`${colors.red}❌ No sources have URL configuration. Cannot test URLs.${colors.reset}`);
294 | process.exit(1);
295 | }
296 |
297 | console.log(`\n${colors.bold}Testing URLs for ${sourcesWithUrls.length} sources with URL configuration...${colors.reset}`);
298 |
299 | // Test each source
300 | const allResults: TestResult[] = [];
301 | for (const library of sourcesWithUrls) {
302 | try {
303 | const results = await validateSourceUrls(library, 5);
304 | allResults.push(...results);
305 | } catch (error) {
306 | console.error(`${colors.red}❌ Error testing ${library.name}: ${error}${colors.reset}`);
307 | }
308 | }
309 |
310 | // Generate summary report
311 | await generateSummaryReport(allResults);
312 |
313 | // Exit with appropriate code
314 | const hasFailures = allResults.some(r => !r.ok);
315 | if (hasFailures) {
316 | console.log(`\n${colors.yellow}⚠️ Some URLs failed validation. Check the results above.${colors.reset}`);
317 | process.exit(1);
318 | } else {
319 | console.log(`\n${colors.green}🎉 All URLs validated successfully!${colors.reset}`);
320 | process.exit(0);
321 | }
322 |
323 | } catch (error) {
324 | console.error(`${colors.red}❌ Error: ${error}${colors.reset}`);
325 | process.exit(1);
326 | }
327 | }
328 |
329 | // Handle CLI usage
330 | if (process.argv[1] === fileURLToPath(import.meta.url)) {
331 | main().catch(console.error);
332 | }
333 |
334 |
```
--------------------------------------------------------------------------------
/.github/workflows/deploy-mcp-sap-docs.yml:
--------------------------------------------------------------------------------
```yaml
1 | name: Deploy MCP stack
2 |
3 | on:
4 | push:
5 | branches: [ main ]
6 |
7 | # Allow manual triggering
8 | workflow_dispatch:
9 |
10 | jobs:
11 | deploy:
12 | runs-on: ubuntu-22.04
13 | # 👇 must match the environment where you stored the secrets
14 | environment:
15 | name: remove server
16 |
17 | steps:
18 | - name: Check out repo (for action context only)
19 | uses: actions/checkout@v4
20 | with:
21 | # Fetch more than just the triggering commit to handle workflow reruns
22 | fetch-depth: 10
23 |
24 | - name: Ensure we have the latest main branch
25 | run: |
26 | git fetch origin main
27 | git checkout main
28 | git reset --hard origin/main
29 | echo "Current HEAD: $(git rev-parse HEAD)"
30 | echo "Latest main: $(git rev-parse origin/main)"
31 |
32 | - name: Preflight verify required secrets are present
33 | run: |
34 | set -euo pipefail
35 | for s in SERVER_IP SERVER_USERNAME SSH_PRIVATE_KEY; do
36 | [ -n "${!s}" ] || { echo "Missing $s"; exit 1; }
37 | done
38 | env:
39 | SERVER_IP: ${{ secrets.SERVER_IP }}
40 | SERVER_USERNAME: ${{ secrets.SERVER_USERNAME }}
41 | SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
42 |
43 | - name: Auto-increment version
44 | run: |
45 | # Get current version and increment patch
46 | CURRENT_VERSION=$(node -p "require('./package.json').version")
47 | echo "Current version: $CURRENT_VERSION"
48 |
49 | # Extract major.minor.patch
50 | IFS='.' read -ra PARTS <<< "$CURRENT_VERSION"
51 | MAJOR=${PARTS[0]}
52 | MINOR=${PARTS[1]}
53 | PATCH=${PARTS[2]}
54 |
55 | # Increment patch version
56 | NEW_PATCH=$((PATCH + 1))
57 | NEW_VERSION="$MAJOR.$MINOR.$NEW_PATCH"
58 |
59 | echo "New version: $NEW_VERSION"
60 | echo "NEW_VERSION=$NEW_VERSION" >> $GITHUB_ENV
61 |
62 | # Update package.json
63 | npm version $NEW_VERSION --no-git-tag-version
64 |
65 | - name: Update hardcoded version in streamable server
66 | run: |
67 | # Update the hardcoded version in streamable-http-server.ts
68 | sed -i 's/const VERSION = "[0-9]*\.[0-9]*\.[0-9]*";/const VERSION = "'${{ env.NEW_VERSION }}'";/g' src/streamable-http-server.ts
69 |
70 | - name: Commit version bump
71 | id: version_bump
72 | continue-on-error: true
73 | run: |
74 | set -e
75 |
76 | git config --local user.email "[email protected]"
77 | git config --local user.name "GitHub Action"
78 |
79 | # Check if there are any changes to commit
80 | git add package.json package-lock.json src/streamable-http-server.ts
81 | if ! git diff --staged --quiet; then
82 |
83 | if git commit -m "chore: bump version to ${{ env.NEW_VERSION }} [skip ci]"; then
84 | echo "Version bump committed successfully"
85 |
86 | # Handle potential conflicts from concurrent pushes or workflow reruns
87 | echo "Attempting to push version bump..."
88 | if ! git push; then
89 | echo "Push failed, likely due to remote changes. Pulling and retrying..."
90 | git fetch origin main
91 |
92 | # Try rebase first
93 | if git rebase origin/main; then
94 | echo "Rebase successful, pushing again..."
95 | git push
96 | else
97 | echo "Rebase failed, attempting merge strategy..."
98 | git rebase --abort
99 | git merge origin/main --no-edit
100 | git push
101 | fi
102 | fi
103 | echo "✅ Version bump pushed successfully"
104 | else
105 | echo "ℹ️ No version changes to commit"
106 | fi
107 | else
108 | echo "ℹ️ No version changes detected"
109 | fi
110 |
111 | - name: Handle version bump failure
112 | if: steps.version_bump.outcome == 'failure'
113 | run: |
114 | echo "⚠️ Version bump failed, but continuing with deployment"
115 | echo "This can happen with concurrent workflows or when rerunning failed deployments"
116 | echo "The deployment will proceed with the current version"
117 |
118 | # Reset any partial git state
119 | git reset --hard HEAD
120 | git clean -fd
121 |
122 | - name: Deploy to server via SSH
123 | uses: appleboy/[email protected]
124 | env:
125 | NEW_VERSION: ${{ env.NEW_VERSION }}
126 | with:
127 | host: ${{ secrets.SERVER_IP }}
128 | username: ${{ secrets.SERVER_USERNAME }}
129 | key: ${{ secrets.SSH_PRIVATE_KEY }}
130 | envs: NEW_VERSION
131 | script: |
132 | set -Eeuo pipefail
133 |
134 | echo "==> Database Health Pre-Check"
135 | cd /opt/mcp-sap/mcp-sap-docs || { echo "Directory not found, will be created"; }
136 |
137 | # Function to check SQLite database integrity
138 | check_db_integrity() {
139 | local db_path="$1"
140 | if [ -f "$db_path" ]; then
141 | echo "🔍 Checking database integrity: $db_path"
142 | if sqlite3 "$db_path" "PRAGMA integrity_check;" 2>/dev/null | grep -q "ok"; then
143 | echo "✅ Database integrity OK"
144 | return 0
145 | else
146 | echo "❌ Database corruption detected"
147 | return 1
148 | fi
149 | else
150 | echo "ℹ️ Database file does not exist: $db_path"
151 | return 1
152 | fi
153 | }
154 |
155 | # Check existing database and create backup
156 | DB_PATH="/opt/mcp-sap/mcp-sap-docs/dist/data/docs.sqlite"
157 | if [ -f "$DB_PATH" ]; then
158 | if ! check_db_integrity "$DB_PATH"; then
159 | echo "==> Database corruption detected - will rebuild"
160 | rm -f "$DB_PATH"
161 | else
162 | echo "==> Creating database backup before deployment"
163 | BACKUP_PATH="/opt/mcp-sap/backups/deploy-backup-$(date +%Y%m%d-%H%M%S).sqlite"
164 | mkdir -p /opt/mcp-sap/backups
165 | cp "$DB_PATH" "$BACKUP_PATH"
166 | echo "✅ Database backed up to $BACKUP_PATH"
167 |
168 | # Keep only last 5 backups
169 | ls -t /opt/mcp-sap/backups/deploy-backup-*.sqlite 2>/dev/null | tail -n +6 | xargs -r rm --
170 | fi
171 | fi
172 |
173 | echo "==> Ensure base path exists and owned by user"
174 | sudo mkdir -p /opt/mcp-sap
175 | sudo chown -R "$USER":"$USER" /opt/mcp-sap
176 |
177 | echo "==> Clone or update repo (with submodules)"
178 | if [ -d /opt/mcp-sap/mcp-sap-docs/.git ]; then
179 | cd /opt/mcp-sap/mcp-sap-docs
180 | git config --global url."https://github.com/".insteadOf [email protected]:
181 | git fetch --prune
182 | git reset --hard origin/main
183 | else
184 | cd /opt/mcp-sap
185 | git config --global url."https://github.com/".insteadOf [email protected]:
186 | git clone https://github.com/marianfoo/mcp-sap-docs.git
187 | cd mcp-sap-docs
188 | fi
189 |
190 | echo "==> Deploying version: $NEW_VERSION"
191 |
192 | echo "==> Configure BM25 search environment"
193 | # Ensure metadata file exists for centralized configuration
194 | [ -f /opt/mcp-sap/mcp-sap-docs/data/metadata.json ] || echo "Metadata file will be created during build"
195 |
196 | echo "==> Check system resources before build"
197 | AVAILABLE_MB=$(df /opt/mcp-sap --output=avail -m | tail -n1)
198 | if [ "$AVAILABLE_MB" -lt 1000 ]; then
199 | echo "❌ ERROR: Insufficient disk space. Available: ${AVAILABLE_MB}MB, Required: 1000MB"
200 | exit 1
201 | fi
202 | echo "✅ Disk space OK: ${AVAILABLE_MB}MB available"
203 |
204 | AVAILABLE_KB=$(awk '/MemAvailable/ { print $2 }' /proc/meminfo)
205 | AVAILABLE_MB_MEM=$((AVAILABLE_KB / 1024))
206 | if [ "$AVAILABLE_MB_MEM" -lt 512 ]; then
207 | echo "❌ ERROR: Insufficient memory. Available: ${AVAILABLE_MB_MEM}MB, Required: 512MB"
208 | exit 1
209 | fi
210 | echo "✅ Memory OK: ${AVAILABLE_MB_MEM}MB available"
211 |
212 | echo "==> Stop MCP services gracefully before build"
213 | pm2 stop mcp-sap-proxy mcp-sap-http mcp-sap-streamable || true
214 | sleep 3
215 |
216 | echo "==> Run setup (shallow, single-branch submodules + build)"
217 | SKIP_NESTED_SUBMODULES=1 bash setup.sh
218 |
219 | echo "==> Verify database integrity after build"
220 | if ! check_db_integrity "$DB_PATH"; then
221 | echo "❌ ERROR: Database corruption after build - deployment failed"
222 | exit 1
223 | fi
224 | echo "✅ Database integrity verified after build"
225 |
226 | echo "==> Create logs directory with proper permissions"
227 | mkdir -p /opt/mcp-sap/logs
228 | chown -R "$USER":"$USER" /opt/mcp-sap/logs
229 |
230 | echo "==> (Re)start MCP services with BM25 search support"
231 | # Proxy (SSE) on 127.0.0.1:18080; HTTP status on 127.0.0.1:3001; Streamable HTTP on 127.0.0.1:3122
232 | pm2 start /opt/mcp-sap/mcp-sap-docs/ecosystem.config.cjs --only mcp-sap-proxy || pm2 restart mcp-sap-proxy
233 | pm2 start /opt/mcp-sap/mcp-sap-docs/ecosystem.config.cjs --only mcp-sap-http || pm2 restart mcp-sap-http
234 | pm2 start /opt/mcp-sap/mcp-sap-docs/ecosystem.config.cjs --only mcp-sap-streamable || pm2 restart mcp-sap-streamable
235 | pm2 save
236 |
237 | echo "==> Enhanced health checks with database verification"
238 | sleep 5
239 |
240 | for i in $(seq 1 30); do curl -fsS http://127.0.0.1:18080/status >/dev/null && break || sleep 2; done
241 | curl -fsS http://127.0.0.1:18080/status
242 | for i in $(seq 1 30); do curl -fsS http://127.0.0.1:3001/status >/dev/null && break || sleep 2; done
243 | curl -fsS http://127.0.0.1:3001/status
244 | # Streamable HTTP server health on 127.0.0.1:3122
245 | for i in $(seq 1 30); do curl -fsS http://127.0.0.1:3122/health >/dev/null && break || sleep 2; done
246 | curl -fsS http://127.0.0.1:3122/health
247 |
248 | # Test actual search functionality to ensure no SQLite corruption
249 | echo "==> Testing search functionality"
250 | SEARCH_TEST=$(curl -s -X POST http://127.0.0.1:3001/mcp -H "Content-Type: application/json" -d '{"role": "user", "content": "test search"}')
251 | if echo "$SEARCH_TEST" | grep -q "SqliteError\|SQLITE_CORRUPT\|Tool execution failed"; then
252 | echo "❌ ERROR: Search test failed - possible database corruption"
253 | echo "Response: $SEARCH_TEST"
254 | exit 1
255 | fi
256 | echo "✅ Search functionality verified - no corruption detected"
257 |
258 | echo "==> Final database integrity check"
259 | if ! check_db_integrity "$DB_PATH"; then
260 | echo "❌ WARNING: Database corruption detected after deployment"
261 | # Don't fail deployment, but alert
262 | else
263 | echo "✅ Final database integrity check passed"
264 | fi
265 |
266 | echo "✅ Deployment completed successfully - Version: $NEW_VERSION"
```
--------------------------------------------------------------------------------
/src/http-server.ts:
--------------------------------------------------------------------------------
```typescript
1 | import { createServer } from "http";
2 | import { readFileSync, statSync, existsSync, readdirSync } from "fs";
3 | import { fileURLToPath } from "url";
4 | import { dirname, join, resolve } from "path";
5 | import { execSync } from "child_process";
6 | import { searchLibraries } from "./lib/localDocs.js";
7 | import { search } from "./lib/search.js";
8 | import { CONFIG } from "./lib/config.js";
9 | import { loadMetadata, getDocUrlConfig } from "./lib/metadata.js";
10 | import { generateDocumentationUrl, formatSearchResult } from "./lib/url-generation/index.js";
11 |
12 | const __filename = fileURLToPath(import.meta.url);
13 | const __dirname = dirname(__filename);
14 |
15 | // ---- build/package meta -----------------------------------------------------
16 | let packageInfo: { version: string; name: string } = { version: "unknown", name: "mcp-sap-docs" };
17 | try {
18 | const packagePath = join(__dirname, "../../package.json");
19 | packageInfo = JSON.parse(readFileSync(packagePath, "utf8"));
20 | } catch (error) {
21 | console.warn("Could not read package.json:", error instanceof Error ? error.message : "Unknown error");
22 | }
23 | const buildTimestamp = new Date().toISOString();
24 |
25 | // ---- helpers ----------------------------------------------------------------
26 | function safeExec(cmd: string, cwd?: string) {
27 | try {
28 | return execSync(cmd, { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], cwd }).trim();
29 | } catch {
30 | return "";
31 | }
32 | }
33 |
34 | // Handle both normal repos and submodules where `.git` is a FILE with `gitdir: …`
35 | function resolveGitDir(repoPath: string): string | null {
36 | const dotGit = join(repoPath, ".git");
37 | if (!existsSync(dotGit)) return null;
38 | const st = statSync(dotGit);
39 | if (st.isDirectory()) return dotGit;
40 |
41 | // .git is a file that points to the real gitdir
42 | const content = readFileSync(dotGit, "utf8");
43 | const m = content.match(/^gitdir:\s*(.+)$/m);
44 | if (!m) return null;
45 | return resolve(repoPath, m[1]);
46 | }
47 |
48 | function readGitMeta(repoPath: string) {
49 | try {
50 | const gitDir = resolveGitDir(repoPath);
51 | if (!gitDir) return { error: "No git dir" };
52 |
53 | const headPath = join(gitDir, "HEAD");
54 | const head = readFileSync(headPath, "utf8").trim();
55 | if (head.startsWith("ref: ")) {
56 | const ref = head.slice(5).trim();
57 | const refPath = join(gitDir, ref);
58 | const commit = readFileSync(refPath, "utf8").trim();
59 | const date = safeExec(`git log -1 --format="%ci"`, repoPath);
60 | return {
61 | branch: ref.split("/").pop(),
62 | commit: commit.substring(0, 7),
63 | fullCommit: commit,
64 | lastModified: date ? new Date(date).toISOString() : statSync(refPath).mtime.toISOString(),
65 | };
66 | } else {
67 | // detached
68 | const date = safeExec(`git log -1 --format="%ci"`, repoPath);
69 | return {
70 | commit: head.substring(0, 7),
71 | fullCommit: head,
72 | detached: true,
73 | lastModified: date ? new Date(date).toISOString() : statSync(headPath).mtime.toISOString(),
74 | };
75 | }
76 | } catch (e: any) {
77 | return { error: e?.message || "git meta error" };
78 | }
79 | }
80 |
81 | // Format results to be MCP-tool compatible, keep legacy formatting
82 | async function handleMCPRequest(content: string) {
83 | try {
84 | // Use simple BM25 search with centralized config
85 | const results = await search(content, {
86 | k: CONFIG.RETURN_K
87 | });
88 |
89 | if (results.length === 0) {
90 | return {
91 | role: "assistant",
92 | content: `No results found for "${content}". Try searching for UI5 controls like 'button', 'table', 'wizard', testing topics like 'wdi5', 'testing', 'e2e', or concepts like 'routing', 'annotation', 'authentication'.`
93 | };
94 | }
95 |
96 | // Format results with URL generation
97 | const formattedResults = results.map((r, index) => {
98 | return formatSearchResult(r, CONFIG.EXCERPT_LENGTH_MAIN, {
99 | generateDocumentationUrl,
100 | getDocUrlConfig
101 | });
102 | }).join('\n');
103 |
104 | const summary = `Found ${results.length} results for '${content}':\n\n${formattedResults}`;
105 |
106 | return { role: "assistant", content: summary };
107 | } catch (error) {
108 | console.error('Hybrid search failed, falling back to original search:', error);
109 | // Fallback to original search
110 | try {
111 | const searchResult = await searchLibraries(content);
112 | if (searchResult.results.length > 0) {
113 | return { role: "assistant", content: searchResult.results[0].description };
114 | }
115 | return {
116 | role: "assistant",
117 | content: searchResult.error || `No results for "${content}". Try 'button', 'table', 'wizard', 'routing', 'annotation', 'authentication', 'cds entity', 'wdi5 testing'.`,
118 | };
119 | } catch (fallbackError) {
120 | console.error("Search error:", error);
121 | return { role: "assistant", content: `Error searching for "${content}". Try a different query.` };
122 | }
123 | }
124 | }
125 |
126 | function json(res: any, code: number, payload: unknown) {
127 | res.statusCode = code;
128 | res.setHeader("Content-Type", "application/json");
129 | res.end(JSON.stringify(payload, null, 2));
130 | }
131 |
132 | // ---- server -----------------------------------------------------------------
133 | const server = createServer(async (req, res) => {
134 | // CORS (you can tighten later if needed)
135 | res.setHeader("Access-Control-Allow-Origin", "*");
136 | res.setHeader("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
137 | res.setHeader("Access-Control-Allow-Headers", "Content-Type");
138 | if (req.method === "OPTIONS") return json(res, 200, { ok: true });
139 |
140 | // healthz/readyz: cheap checks for PM2/K8s or manual curl
141 | if (req.method === "GET" && (req.url === "/healthz" || req.url === "/readyz")) {
142 | return json(res, 200, { status: "ok", ts: new Date().toISOString() });
143 | }
144 |
145 | // status: richer info
146 | if (req.method === "GET" && req.url === "/status") {
147 | // top-level repo git info
148 | let gitInfo: any = {};
149 | try {
150 | const repoPath = resolve(__dirname, "../..");
151 | gitInfo = readGitMeta(repoPath);
152 | // normalize to include branch if unknown
153 | if (!gitInfo.branch) {
154 | const branch = safeExec("git rev-parse --abbrev-ref HEAD", repoPath);
155 | if (branch && branch !== "HEAD") gitInfo.branch = branch;
156 | }
157 | } catch {
158 | gitInfo = { error: "Git info not available" };
159 | }
160 |
161 | // docs/search status
162 | const sourcesRoot = join(__dirname, "../../sources");
163 | const knownSources = [
164 | "sapui5-docs",
165 | "cap-docs",
166 | "openui5",
167 | "wdi5",
168 | "ui5-tooling",
169 | "cloud-mta-build-tool",
170 | "ui5-webcomponents",
171 | "cloud-sdk",
172 | "cloud-sdk-ai"
173 | ];
174 | const presentSources = existsSync(sourcesRoot)
175 | ? readdirSync(sourcesRoot, { withFileTypes: true })
176 | .filter((e) => e.isDirectory())
177 | .map((e) => e.name)
178 | : [];
179 |
180 | const toCheck = knownSources.filter((s) => presentSources.includes(s));
181 | const resources: Record<string, any> = {};
182 | let totalResources = 0;
183 |
184 | for (const name of knownSources) {
185 | const p = join(sourcesRoot, name);
186 | if (!existsSync(p)) {
187 | resources[name] = { status: "missing", error: "not found" };
188 | continue;
189 | }
190 | const meta = readGitMeta(p);
191 | if ((meta as any).error) {
192 | // still count as available content; just git meta missing (e.g., copied tree)
193 | resources[name] = { status: "available", note: (meta as any).error, path: p };
194 | totalResources++;
195 | } else {
196 | resources[name] = { status: "available", path: p, ...meta };
197 | totalResources++;
198 | }
199 | }
200 |
201 | // index + FTS footprint
202 | const dataRoot = join(__dirname, "../../data");
203 | const indexJson = join(dataRoot, "index.json");
204 | const ftsDb = join(dataRoot, "docs.sqlite");
205 | const indexStat = existsSync(indexJson) ? statSync(indexJson) : null;
206 | const ftsStat = existsSync(ftsDb) ? statSync(ftsDb) : null;
207 |
208 | // quick search smoke test
209 | let docsStatus = "unknown";
210 | try {
211 | const testSearch = await searchLibraries("button");
212 | docsStatus = testSearch.results.length > 0 ? "available" : "no_results";
213 | } catch {
214 | docsStatus = "error";
215 | }
216 |
217 | const statusResponse = {
218 | status: "healthy",
219 | service: packageInfo.name,
220 | version: packageInfo.version,
221 | timestamp: new Date().toISOString(),
222 | buildTimestamp,
223 | git: gitInfo,
224 | documentation: {
225 | status: docsStatus,
226 | searchAvailable: true,
227 | communityAvailable: true,
228 | resources: {
229 | totalResources,
230 | sources: resources,
231 | lastUpdated:
232 | Object.values(resources)
233 | .map((s: any) => s.lastModified)
234 | .filter(Boolean)
235 | .sort()
236 | .pop() || "unknown",
237 | artifacts: {
238 | indexJson: indexStat
239 | ? { path: indexJson, sizeBytes: indexStat.size, mtime: indexStat.mtime.toISOString() }
240 | : "missing",
241 | ftsSqlite: ftsStat
242 | ? { path: ftsDb, sizeBytes: ftsStat.size, mtime: ftsStat.mtime.toISOString() }
243 | : "missing",
244 | },
245 | },
246 | },
247 | deployment: {
248 | method: process.env.DEPLOYMENT_METHOD || "unknown",
249 | timestamp: process.env.DEPLOYMENT_TIMESTAMP || "unknown",
250 | triggeredBy: process.env.GITHUB_ACTOR || "unknown",
251 | },
252 | runtime: {
253 | uptimeSeconds: process.uptime(),
254 | nodeVersion: process.version,
255 | platform: process.platform,
256 | pid: process.pid,
257 | port: Number(process.env.PORT || 3001),
258 | bind: "127.0.0.1",
259 | },
260 | };
261 |
262 | return json(res, 200, statusResponse);
263 | }
264 |
265 | // Legacy SSE endpoint - redirect to MCP
266 | if (req.url === "/sse") {
267 | const redirectInfo = {
268 | error: "SSE endpoint deprecated",
269 | message: "The /sse endpoint has been removed. Please use the modern /mcp endpoint instead.",
270 | migration: {
271 | old_endpoint: "/sse",
272 | new_endpoint: "/mcp",
273 | transport: "MCP Streamable HTTP",
274 | protocol_version: "2025-07-09"
275 | },
276 | documentation: "https://github.com/marianfoo/mcp-sap-docs#connect-from-your-mcp-client",
277 | alternatives: {
278 | "Local MCP Streamable HTTP": "http://127.0.0.1:3122/mcp",
279 | "Public MCP Streamable HTTP": "https://mcp-sap-docs.marianzeis.de/mcp"
280 | }
281 | };
282 |
283 | res.setHeader("Content-Type", "application/json");
284 | return json(res, 410, redirectInfo);
285 | }
286 |
287 | if (req.method === "POST" && req.url === "/mcp") {
288 | let body = "";
289 | req.on("data", (chunk) => (body += chunk.toString()));
290 | req.on("end", async () => {
291 | try {
292 | const mcpRequest: { role: string; content: string } = JSON.parse(body);
293 | const response = await handleMCPRequest(mcpRequest.content);
294 | return json(res, 200, response);
295 | } catch {
296 | return json(res, 400, { error: "Invalid JSON" });
297 | }
298 | });
299 | return;
300 | }
301 |
302 | // default 404 JSON (keeps curl|jq friendly)
303 | return json(res, 404, { error: "Not Found", path: req.url, method: req.method });
304 | });
305 |
306 | // Initialize search system with metadata
307 | (async () => {
308 | console.log('🔧 Initializing BM25 search system...');
309 | try {
310 | loadMetadata();
311 | console.log('✅ Search system ready with metadata');
312 | } catch (error) {
313 | console.warn('⚠️ Metadata loading failed, using defaults');
314 | console.log('✅ Search system ready');
315 | }
316 |
317 | // Start server
318 | const PORT = Number(process.env.PORT || 3001);
319 | // Bind to 127.0.0.1 to keep local-only
320 | server.listen(PORT, "127.0.0.1", () => {
321 | console.log(`📚 HTTP server running on http://127.0.0.1:${PORT} (status: /status, health: /healthz, ready: /readyz)`);
322 | });
323 | })();
```
--------------------------------------------------------------------------------
/src/streamable-http-server.ts:
--------------------------------------------------------------------------------
```typescript
1 | import express, { Request, Response } from "express";
2 | import { randomUUID } from "node:crypto";
3 | import cors from "cors";
4 | import { Server } from "@modelcontextprotocol/sdk/server/index.js";
5 | import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
6 | import {
7 | isInitializeRequest
8 | } from "@modelcontextprotocol/sdk/types.js";
9 | import { logger } from "./lib/logger.js";
10 | import { BaseServerHandler } from "./lib/BaseServerHandler.js";
11 |
12 | // Version will be updated by deployment script
13 | const VERSION = "0.3.19";
14 |
15 |
16 | // Simple in-memory event store for resumability
17 | class InMemoryEventStore {
18 | private events: Map<string, Array<{ eventId: string; message: any }>> = new Map();
19 | private eventCounter = 0;
20 |
21 | async storeEvent(streamId: string, message: any): Promise<string> {
22 | const eventId = `event-${this.eventCounter++}`;
23 |
24 | if (!this.events.has(streamId)) {
25 | this.events.set(streamId, []);
26 | }
27 |
28 | this.events.get(streamId)!.push({ eventId, message });
29 |
30 | // Keep only last 100 events per stream to prevent memory issues
31 | const streamEvents = this.events.get(streamId)!;
32 | if (streamEvents.length > 100) {
33 | streamEvents.splice(0, streamEvents.length - 100);
34 | }
35 |
36 | return eventId;
37 | }
38 |
39 | async replayEventsAfter(lastEventId: string, { send }: { send: (eventId: string, message: any) => Promise<void> }): Promise<string> {
40 | // Find the stream that contains this event ID
41 | for (const [streamId, events] of this.events.entries()) {
42 | const eventIndex = events.findIndex(e => e.eventId === lastEventId);
43 | if (eventIndex !== -1) {
44 | // Replay all events after the specified event ID
45 | for (let i = eventIndex + 1; i < events.length; i++) {
46 | const event = events[i];
47 | await send(event.eventId, event.message);
48 | }
49 | return streamId;
50 | }
51 | }
52 |
53 | // If event ID not found, return a new stream ID
54 | return `stream-${randomUUID()}`;
55 | }
56 | }
57 |
58 | function createServer() {
59 | const serverOptions: NonNullable<ConstructorParameters<typeof Server>[1]> & {
60 | protocolVersions?: string[];
61 | } = {
62 | protocolVersions: ["2025-07-09"],
63 | capabilities: {
64 | // resources: {}, // DISABLED: Causes 60,000+ resources which breaks Cursor
65 | tools: {} // Enable tools capability
66 | }
67 | };
68 |
69 | const srv = new Server({
70 | name: "SAP Docs Streamable HTTP",
71 | description:
72 | "SAP documentation server with Streamable HTTP transport - supports SAPUI5, CAP, wdi5, SAP Community, SAP Help Portal, and ABAP Keyword Documentation integration",
73 | version: VERSION
74 | }, serverOptions);
75 |
76 | // Configure server with shared handlers
77 | BaseServerHandler.configureServer(srv);
78 |
79 | return srv;
80 | }
81 |
82 | async function main() {
83 | // Initialize search system with metadata
84 | BaseServerHandler.initializeMetadata();
85 |
86 | const MCP_PORT = process.env.MCP_PORT ? parseInt(process.env.MCP_PORT, 10) : 3122;
87 |
88 | // Create Express application
89 | const app = express();
90 | app.use(express.json());
91 |
92 | // Configure CORS to expose Mcp-Session-Id header for browser-based clients
93 | app.use(cors({
94 | origin: '*', // Allow all origins - adjust as needed for production
95 | exposedHeaders: ['Mcp-Session-Id']
96 | }));
97 |
98 | // Store transports by session ID
99 | const transports: { [sessionId: string]: StreamableHTTPServerTransport } = {};
100 |
101 | // Create event store for resumability
102 | const eventStore = new InMemoryEventStore();
103 |
104 | // Legacy SSE endpoint - redirect to MCP
105 | app.all('/sse', (req: Request, res: Response) => {
106 | const redirectInfo = {
107 | error: "SSE endpoint deprecated",
108 | message: "The /sse endpoint has been removed. Please use the modern /mcp endpoint instead.",
109 | migration: {
110 | old_endpoint: "/sse",
111 | new_endpoint: "/mcp",
112 | transport: "MCP Streamable HTTP",
113 | protocol_version: "2025-07-09"
114 | },
115 | documentation: "https://github.com/marianfoo/mcp-sap-docs#connect-from-your-mcp-client",
116 | alternatives: {
117 | "Local MCP Streamable HTTP": "http://127.0.0.1:3122/mcp",
118 | "Public MCP Streamable HTTP": "https://mcp-sap-docs.marianzeis.de/mcp"
119 | }
120 | };
121 |
122 | res.status(410).json(redirectInfo);
123 | });
124 |
125 | // Handle all MCP Streamable HTTP requests (GET, POST, DELETE) on a single endpoint
126 | app.all('/mcp', async (req: Request, res: Response) => {
127 | const requestId = `http_${Date.now()}_${Math.random().toString(36).substr(2, 6)}`;
128 | logger.debug(`Received ${req.method} request to /mcp`, {
129 | requestId,
130 | userAgent: req.headers['user-agent'],
131 | contentLength: req.headers['content-length'],
132 | sessionId: req.headers['mcp-session-id'] as string || 'none'
133 | });
134 |
135 | try {
136 | // Check for existing session ID
137 | const sessionId = req.headers['mcp-session-id'] as string;
138 | let transport: StreamableHTTPServerTransport;
139 |
140 | if (sessionId && transports[sessionId]) {
141 | // Reuse existing transport
142 | transport = transports[sessionId];
143 | logger.logTransportEvent('transport_reused', sessionId, {
144 | requestId,
145 | method: req.method,
146 | transportCount: Object.keys(transports).length
147 | });
148 | } else if (!sessionId && req.method === 'POST' && req.is('application/json') && req.body?.method === 'initialize') {
149 | // New initialization request - create new transport
150 | const cleanupTransport = (
151 | sessionId: string | undefined,
152 | trigger: "onsessionclosed" | "onclose",
153 | context: Record<string, unknown> = {}
154 | ) => {
155 | if (!sessionId) {
156 | return;
157 | }
158 |
159 | const hadTransport = Boolean(transports[sessionId]);
160 |
161 | if (hadTransport) {
162 | delete transports[sessionId];
163 | }
164 |
165 | logger.logTransportEvent("session_closed", sessionId, {
166 | ...context,
167 | trigger,
168 | transportCount: Object.keys(transports).length,
169 | ...(hadTransport ? {} : { note: "session already cleaned up" })
170 | });
171 | };
172 |
173 | transport = new StreamableHTTPServerTransport({
174 | sessionIdGenerator: () => randomUUID(),
175 | eventStore, // Enable resumability
176 | onsessioninitialized: (sessionId: string) => {
177 | // Store the transport by session ID when session is initialized
178 | logger.logTransportEvent('session_initialized', sessionId, {
179 | requestId,
180 | transportCount: Object.keys(transports).length + 1
181 | });
182 | transports[sessionId] = transport;
183 | },
184 | onsessionclosed: (sessionId: string) => {
185 | cleanupTransport(sessionId, 'onsessionclosed');
186 | }
187 | });
188 |
189 | // Set up onclose handler to clean up transport when closed
190 | transport.onclose = () => {
191 | cleanupTransport(transport.sessionId, 'onclose', { requestId });
192 | };
193 |
194 | // Connect the transport to the MCP server
195 | const server = createServer();
196 | await server.connect(transport);
197 |
198 | logger.logTransportEvent('transport_created', undefined, {
199 | requestId,
200 | method: req.method
201 | });
202 | } else {
203 | // Invalid request - no session ID or not initialization request
204 | logger.warn('Invalid MCP request', {
205 | requestId,
206 | method: req.method,
207 | hasSessionId: !!sessionId,
208 | isInitRequest: req.method === 'POST' && req.is('application/json') && req.body?.method === 'initialize',
209 | sessionId: sessionId || 'none',
210 | userAgent: req.headers['user-agent']
211 | });
212 |
213 | res.status(400).json({
214 | jsonrpc: '2.0',
215 | error: {
216 | code: -32000,
217 | message: 'Bad Request: No valid session ID provided or not an initialization request',
218 | },
219 | id: null,
220 | });
221 | return;
222 | }
223 |
224 | // Handle the request with the transport
225 | await transport.handleRequest(req, res, req.body);
226 | } catch (error) {
227 | logger.error('Error handling MCP request', {
228 | requestId,
229 | error: String(error),
230 | stack: error instanceof Error ? error.stack : undefined,
231 | method: req.method,
232 | sessionId: req.headers['mcp-session-id'] as string || 'none',
233 | userAgent: req.headers['user-agent']
234 | });
235 |
236 | if (!res.headersSent) {
237 | res.status(500).json({
238 | jsonrpc: '2.0',
239 | error: {
240 | code: -32603,
241 | message: `Internal server error. Request ID: ${requestId}`,
242 | },
243 | id: null,
244 | });
245 | }
246 | }
247 | });
248 |
249 | // Health check endpoint
250 | app.get('/health', (req: Request, res: Response) => {
251 | res.json({
252 | status: 'healthy',
253 | service: 'mcp-sap-docs-streamable',
254 | version: VERSION,
255 | timestamp: new Date().toISOString(),
256 | transport: 'streamable-http',
257 | protocol: '2025-07-09'
258 | });
259 | });
260 |
261 | // Start the server (bind to localhost for local-only access)
262 | const server = app.listen(MCP_PORT, '127.0.0.1', (error?: Error) => {
263 | if (error) {
264 | console.error('Failed to start server:', error);
265 | process.exit(1);
266 | }
267 | });
268 |
269 | // Configure server timeouts for MCP connections
270 | server.timeout = 0; // Disable HTTP timeout for long-lived MCP connections
271 | server.keepAliveTimeout = 0; // Disable keep-alive timeout
272 | server.headersTimeout = 0; // Disable headers timeout
273 |
274 | console.log(`📚 MCP Streamable HTTP Server listening on http://127.0.0.1:${MCP_PORT}`);
275 | console.log(`
276 | ==============================================
277 | MCP STREAMABLE HTTP SERVER
278 | Protocol version: 2025-07-09
279 |
280 | Endpoint: /mcp
281 | Methods: GET, POST, DELETE
282 | Usage:
283 | - Initialize with POST to /mcp
284 | - Establish stream with GET to /mcp
285 | - Send requests with POST to /mcp
286 | - Terminate session with DELETE to /mcp
287 |
288 | Health check: GET /health
289 | ==============================================
290 | `);
291 |
292 | // Log server startup
293 | logger.info("MCP SAP Docs Streamable HTTP server starting up", {
294 | port: MCP_PORT,
295 | nodeEnv: process.env.NODE_ENV,
296 | logLevel: process.env.LOG_LEVEL,
297 | logFormat: process.env.LOG_FORMAT
298 | });
299 |
300 | // Log successful startup
301 | logger.info("MCP SAP Docs Streamable HTTP server ready", {
302 | transport: "streamable-http",
303 | port: MCP_PORT,
304 | pid: process.pid
305 | });
306 |
307 | // Set up performance monitoring (every 5 minutes)
308 | const performanceInterval = setInterval(() => {
309 | logger.logPerformanceMetrics();
310 | logger.info('Active sessions status', {
311 | activeSessions: Object.keys(transports).length,
312 | sessionIds: Object.keys(transports),
313 | timestamp: new Date().toISOString()
314 | });
315 | }, 5 * 60 * 1000);
316 |
317 | // Handle server shutdown
318 | process.on('SIGINT', async () => {
319 | logger.info('Shutdown signal received, closing server gracefully');
320 |
321 | // Clear performance monitoring
322 | clearInterval(performanceInterval);
323 |
324 | // Close all active transports to properly clean up resources
325 | const sessionIds = Object.keys(transports);
326 | logger.info(`Closing ${sessionIds.length} active sessions`);
327 |
328 | for (const sessionId of sessionIds) {
329 | try {
330 | logger.logTransportEvent('session_shutdown', sessionId);
331 | await transports[sessionId].close();
332 | delete transports[sessionId];
333 | } catch (error) {
334 | logger.error('Error closing transport during shutdown', {
335 | sessionId,
336 | error: String(error)
337 | });
338 | }
339 | }
340 |
341 | logger.info('Server shutdown complete');
342 | process.exit(0);
343 | });
344 | }
345 |
346 | main().catch((e) => {
347 | console.error("Fatal:", e);
348 | process.exit(1);
349 | });
350 |
```
--------------------------------------------------------------------------------
/test/community-search.ts:
--------------------------------------------------------------------------------
```typescript
1 | #!/usr/bin/env node
2 |
3 | // Combined test script for SAP Community Search functionality
4 | // Tests search, batch retrieval, and convenience functions
5 |
6 | import {
7 | searchCommunityBestMatch,
8 | getCommunityPostByUrl,
9 | getCommunityPostsByIds,
10 | getCommunityPostById,
11 | searchAndGetTopPosts
12 | } from '../dist/src/lib/communityBestMatch.js';
13 |
14 | interface TestOptions {
15 | userAgent: string;
16 | delay: number;
17 | }
18 |
19 | const defaultOptions: TestOptions = {
20 | userAgent: 'SAP-Docs-MCP-Test/1.0',
21 | delay: 2000
22 | };
23 |
24 | // Utility function for delays
25 | const delay = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
26 |
27 | // Test 1: Community Search with HTML Scraping
28 | async function testCommunitySearch(options: TestOptions = defaultOptions): Promise<void> {
29 | console.log('🔍 Testing SAP Community Search with HTML Scraping');
30 | console.log('='.repeat(60));
31 |
32 | const testQueries = [
33 | 'odata cache',
34 | 'fiori elements',
35 | 'CAP authentication'
36 | ];
37 |
38 | for (const query of testQueries) {
39 | console.log(`\n📝 Testing query: "${query}"`);
40 | console.log('-'.repeat(40));
41 |
42 | try {
43 | const results = await searchCommunityBestMatch(query, {
44 | includeBlogs: true,
45 | limit: 5,
46 | userAgent: options.userAgent
47 | });
48 |
49 | if (results.length === 0) {
50 | console.log('❌ No results found');
51 | continue;
52 | }
53 |
54 | console.log(`✅ Found ${results.length} results:`);
55 |
56 | results.forEach((result, index) => {
57 | console.log(`\n${index + 1}. ${result.title}`);
58 | console.log(` URL: ${result.url}`);
59 | console.log(` Author: ${result.author || 'Unknown'}`);
60 | console.log(` Published: ${result.published || 'Unknown'}`);
61 | console.log(` Likes: ${result.likes || 0}`);
62 | console.log(` Snippet: ${result.snippet ? result.snippet.substring(0, 100) + '...' : 'No snippet'}`);
63 | console.log(` Tags: ${result.tags?.join(', ') || 'None'}`);
64 | console.log(` Post ID: ${result.postId || 'Not extracted'}`);
65 |
66 | // Verify post ID extraction
67 | if (result.postId) {
68 | console.log(` ✅ Post ID extracted: ${result.postId}`);
69 | } else {
70 | console.log(` ⚠️ Post ID not extracted from URL: ${result.url}`);
71 | }
72 | });
73 |
74 | // Test detailed post retrieval for the first result using URL scraping
75 | if (results.length > 0) {
76 | console.log(`\n🔎 Testing URL-based post retrieval for: "${results[0].title}"`);
77 | console.log('-'.repeat(30));
78 |
79 | try {
80 | const postContent = await getCommunityPostByUrl(results[0].url, options.userAgent);
81 |
82 | if (postContent) {
83 | console.log('✅ Successfully retrieved full post content via URL scraping:');
84 | console.log(postContent.substring(0, 400) + '...\n');
85 | } else {
86 | console.log('❌ Failed to retrieve full post content via URL scraping');
87 | }
88 | } catch (error: any) {
89 | console.log(`❌ Error retrieving post content: ${error.message}`);
90 | }
91 | }
92 |
93 | } catch (error: any) {
94 | console.log(`❌ Error searching for "${query}": ${error.message}`);
95 | }
96 |
97 | // Add delay between requests to be respectful
98 | if (options.delay > 0) {
99 | await delay(options.delay);
100 | }
101 | }
102 | }
103 |
104 | // Test 2: Batch Retrieval with LiQL API
105 | async function testBatchRetrieval(options: TestOptions = defaultOptions): Promise<void> {
106 | console.log('\n\n📦 Testing SAP Community Batch Retrieval with LiQL API');
107 | console.log('='.repeat(60));
108 |
109 | // Test with known post IDs
110 | const testPostIds = ['13961398', '13446100', '14152848'];
111 |
112 | console.log('📦 Testing batch retrieval for multiple posts:');
113 | console.log(`Post IDs: ${testPostIds.join(', ')}`);
114 | console.log('-'.repeat(40));
115 |
116 | try {
117 | const results = await getCommunityPostsByIds(testPostIds, options.userAgent);
118 |
119 | console.log(`✅ Successfully retrieved ${Object.keys(results).length} out of ${testPostIds.length} posts\n`);
120 |
121 | for (const postId of testPostIds) {
122 | if (results[postId]) {
123 | console.log(`✅ Post ${postId}:`);
124 | const content = results[postId];
125 | const lines = content.split('\n');
126 | console.log(` Title: ${lines[0].replace('# ', '')}`);
127 |
128 | // Extract published date
129 | const publishedLine = lines.find(line => line.startsWith('**Published**:'));
130 | if (publishedLine) {
131 | console.log(` ${publishedLine}`);
132 | }
133 |
134 | // Show content preview
135 | const contentStart = content.indexOf('---\n\n') + 5;
136 | const contentEnd = content.lastIndexOf('\n\n---');
137 | if (contentStart > 4 && contentEnd > contentStart) {
138 | const contentPreview = content.slice(contentStart, contentEnd).substring(0, 200) + '...';
139 | console.log(` Content preview: ${contentPreview}`);
140 | }
141 | console.log();
142 | } else {
143 | console.log(`❌ Post ${postId}: Not retrieved`);
144 | }
145 | }
146 |
147 | } catch (error: any) {
148 | console.log(`❌ Batch retrieval failed: ${error.message}`);
149 | }
150 | }
151 |
152 | // Test 3: Single Post Retrieval
153 | async function testSingleRetrieval(options: TestOptions = defaultOptions): Promise<void> {
154 | console.log('\n🎯 Testing single post retrieval via LiQL API');
155 | console.log('='.repeat(60));
156 |
157 | const testPostId = '13961398'; // FIORI Cache Maintenance
158 |
159 | try {
160 | console.log(`Testing single retrieval for post: ${testPostId}`);
161 | const content = await getCommunityPostById(testPostId, options.userAgent);
162 |
163 | if (content) {
164 | console.log('✅ Successfully retrieved single post:');
165 | console.log(content.substring(0, 500) + '...\n');
166 |
167 | // Verify expected content
168 | if (content.includes('FIORI Cache Maintenance')) {
169 | console.log('✅ Title verification successful');
170 | }
171 | if (content.includes('SMICM')) {
172 | console.log('✅ Content verification successful');
173 | }
174 | if (content.includes('2024')) {
175 | console.log('✅ Date verification successful');
176 | }
177 | } else {
178 | console.log('❌ Single retrieval failed - no content returned');
179 | }
180 | } catch (error: any) {
181 | console.log(`❌ Single retrieval failed: ${error.message}`);
182 | }
183 | }
184 |
185 | // Test 4: Direct LiQL API Testing
186 | async function testLiQLAPIDirectly(options: TestOptions = defaultOptions): Promise<void> {
187 | console.log('\n\n🧪 Testing LiQL API directly');
188 | console.log('='.repeat(60));
189 |
190 | const testIds = ['13961398', '13446100'];
191 | const idList = testIds.map(id => `'${id}'`).join(', ');
192 | const liqlQuery = `select body, id, subject, search_snippet, post_time from messages where id in (${idList})`;
193 | const url = `https://community.sap.com/api/2.0/search?q=${encodeURIComponent(liqlQuery)}`;
194 |
195 | console.log(`Testing URL: ${url.substring(0, 120)}...`);
196 |
197 | try {
198 | const response = await fetch(url, {
199 | headers: {
200 | 'Accept': 'application/json',
201 | 'User-Agent': options.userAgent
202 | }
203 | });
204 |
205 | if (!response.ok) {
206 | console.log(`❌ API returned ${response.status}: ${response.statusText}`);
207 | return;
208 | }
209 |
210 | const data = await response.json();
211 | console.log(`✅ API Response status: ${data.status}`);
212 | console.log(`✅ Items returned: ${data.data?.items?.length || 0}`);
213 |
214 | if (data.data?.items) {
215 | for (const item of data.data.items) {
216 | console.log(` - Post ${item.id}: "${item.subject}" (${item.post_time})`);
217 | }
218 | }
219 |
220 | } catch (error: any) {
221 | console.log(`❌ Direct API test failed: ${error.message}`);
222 | }
223 | }
224 |
225 | // Test 5: Convenience Function (Search + Get Top Posts)
226 | async function testConvenienceFunction(options: TestOptions = defaultOptions): Promise<void> {
227 | console.log('\n\n🚀 Testing Search + Get Top Posts Convenience Function');
228 | console.log('='.repeat(60));
229 |
230 | const query = 'odata cache';
231 | const topN = 3;
232 |
233 | console.log(`Query: "${query}"`);
234 | console.log(`Getting top ${topN} posts with full content...\n`);
235 |
236 | try {
237 | const result = await searchAndGetTopPosts(query, topN, {
238 | includeBlogs: true,
239 | userAgent: options.userAgent
240 | });
241 |
242 | console.log(`✅ Search found ${result.search.length} results`);
243 | console.log(`✅ Retrieved full content for ${Object.keys(result.posts).length} posts\n`);
244 |
245 | // Display search results with post content
246 | for (let i = 0; i < result.search.length; i++) {
247 | const searchResult = result.search[i];
248 | const postContent = result.posts[searchResult.postId || ''];
249 |
250 | console.log(`${i + 1}. ${searchResult.title}`);
251 | console.log(` Post ID: ${searchResult.postId}`);
252 | console.log(` URL: ${searchResult.url}`);
253 | console.log(` Author: ${searchResult.author || 'Unknown'}`);
254 | console.log(` Likes: ${searchResult.likes || 0}`);
255 |
256 | if (postContent) {
257 | console.log(' ✅ Full content retrieved:');
258 | const contentPreview = postContent.split('\n\n---\n\n')[1] || postContent;
259 | console.log(` "${contentPreview.substring(0, 150)}..."`);
260 | } else {
261 | console.log(' ❌ Full content not available');
262 | }
263 | console.log();
264 | }
265 |
266 | // Example usage demonstration
267 | console.log('📋 Example: How to use this data');
268 | console.log('='.repeat(40));
269 | console.log('// Search and get top 3 posts about OData cache:');
270 | console.log(`const { search, posts } = await searchAndGetTopPosts('${query}', ${topN});`);
271 | console.log('');
272 | console.log('// Display results:');
273 | console.log('search.forEach((result, index) => {');
274 | console.log(' console.log(`${index + 1}. ${result.title}`);');
275 | console.log(' if (posts[result.postId]) {');
276 | console.log(' console.log(posts[result.postId]); // Full formatted content');
277 | console.log(' }');
278 | console.log('});');
279 |
280 | } catch (error: any) {
281 | console.error(`❌ Test failed: ${error.message}`);
282 | }
283 | }
284 |
285 | // Test 6: Specific Known Post
286 | async function testSpecificPost(options: TestOptions = defaultOptions): Promise<void> {
287 | console.log('\n\n🎯 Testing specific known post retrieval');
288 | console.log('='.repeat(60));
289 |
290 | // Test with the known SAP Community URL
291 | const testUrl = 'https://community.sap.com/t5/technology-blog-posts-by-sap/fiori-cache-maintenance/ba-p/13961398';
292 |
293 | try {
294 | console.log(`Testing URL: ${testUrl}`);
295 | console.log(`Expected Post ID: 13961398`);
296 |
297 | const content = await getCommunityPostByUrl(testUrl, options.userAgent);
298 |
299 | if (content) {
300 | console.log('✅ Successfully retrieved content:');
301 | console.log(content.substring(0, 600) + '...');
302 |
303 | // Verify the content contains expected elements
304 | if (content.includes('FIORI Cache Maintenance')) {
305 | console.log('✅ Title extraction successful');
306 | }
307 | if (content.includes('MarkNed')) {
308 | console.log('✅ Author extraction successful');
309 | }
310 | if (content.includes('SMICM')) {
311 | console.log('✅ Content extraction successful');
312 | }
313 | } else {
314 | console.log('❌ No content retrieved');
315 | }
316 | } catch (error: any) {
317 | console.log(`❌ Error: ${error.message}`);
318 | }
319 | }
320 |
321 | // Main test runner
322 | async function main(): Promise<void> {
323 | console.log('🚀 SAP Community Search - Comprehensive Test Suite');
324 | console.log('==================================================');
325 | console.log(`Started at: ${new Date().toISOString()}\n`);
326 |
327 | const options: TestOptions = {
328 | userAgent: 'SAP-Docs-MCP-Test/1.0',
329 | delay: 1500 // Reduced delay for faster testing
330 | };
331 |
332 | try {
333 | // Run all tests sequentially
334 | await testCommunitySearch(options);
335 | await testBatchRetrieval(options);
336 | await testSingleRetrieval(options);
337 | await testLiQLAPIDirectly(options);
338 | await testConvenienceFunction(options);
339 | await testSpecificPost(options);
340 |
341 | console.log('\n\n🎉 All tests completed successfully!');
342 | console.log('=====================================');
343 | console.log(`Finished at: ${new Date().toISOString()}`);
344 |
345 | } catch (error: any) {
346 | console.error('❌ Test suite failed:', error);
347 | process.exit(1);
348 | }
349 | }
350 |
351 | // Handle graceful shutdown
352 | process.on('SIGINT', () => {
353 | console.log('\n👋 Test interrupted by user');
354 | process.exit(0);
355 | });
356 |
357 | // Run the tests if this file is executed directly
358 | if (import.meta.url === `file://${process.argv[1]}`) {
359 | main().catch(error => {
360 | console.error('💥 Unexpected error:', error);
361 | process.exit(1);
362 | });
363 | }
364 |
365 | // Export functions for potential use in other test files
366 | export {
367 | testCommunitySearch,
368 | testBatchRetrieval,
369 | testSingleRetrieval,
370 | testLiQLAPIDirectly,
371 | testConvenienceFunction,
372 | testSpecificPost,
373 | main as runAllTests
374 | };
```
--------------------------------------------------------------------------------
/src/lib/communityBestMatch.ts:
--------------------------------------------------------------------------------
```typescript
1 | // src/lib/communityBestMatch.ts
2 | // Scrape SAP Community search "Best Match" results directly from the HTML page.
3 | // No external dependencies; best-effort selectors based on current Khoros layout.
4 |
5 | import { CONFIG } from "./config.js";
6 | import { truncateContent } from "./truncate.js";
7 |
8 | export interface BestMatchHit {
9 | title: string;
10 | url: string;
11 | author?: string;
12 | published?: string; // e.g., "2024 Dec 11 4:31 PM"
13 | likes?: number;
14 | snippet?: string;
15 | tags?: string[];
16 | postId?: string; // extracted from URL for retrieval
17 | }
18 |
19 | type Options = {
20 | includeBlogs?: boolean; // default true
21 | limit?: number; // default 20
22 | userAgent?: string; // optional UA override
23 | };
24 |
25 | const BASE = "https://community.sap.com";
26 |
27 | const buildSearchUrl = (q: string, includeBlogs = true) => {
28 | const params = new URLSearchParams({
29 | collapse_discussion: "true",
30 | q,
31 | });
32 | if (includeBlogs) {
33 | params.set("filter", "includeBlogs");
34 | params.set("include_blogs", "true");
35 | }
36 | // "tab/message" view surfaces posts sorted by Best Match by default
37 | return `${BASE}/t5/forums/searchpage/tab/message?${params.toString()}`;
38 | };
39 |
40 | const decodeEntities = (s = "") =>
41 | s
42 | .replace(/&/g, "&")
43 | .replace(/</g, "<")
44 | .replace(/>/g, ">")
45 | .replace(/"/g, '"')
46 | .replace(/'/g, "'");
47 |
48 | const stripTags = (html = "") =>
49 | decodeEntities(html.replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim());
50 |
51 | const absolutize = (href: string) =>
52 | href?.startsWith("http") ? href : new URL(href, BASE).href;
53 |
54 | // Extract post ID from URL for later retrieval
55 | const extractPostId = (url: string): string | undefined => {
56 | // Extract from URL patterns like: /ba-p/13961398 or /td-p/13961398
57 | const urlMatch = url.match(/\/(?:ba-p|td-p)\/(\d+)/);
58 | if (urlMatch) {
59 | return urlMatch[1];
60 | }
61 |
62 | // Fallback: extract from end of URL
63 | const endMatch = url.match(/\/(\d+)(?:\?|$)/);
64 | return endMatch ? endMatch[1] : undefined;
65 | };
66 |
67 | async function fetchText(url: string, userAgent?: string) {
68 | const res = await fetch(url, {
69 | headers: {
70 | "User-Agent": userAgent || "sap-docs-mcp/1.0 (BestMatchScraper)",
71 | "Accept": "text/html,application/xhtml+xml",
72 | },
73 | });
74 | if (!res.ok) throw new Error(`${url} -> ${res.status} ${res.statusText}`);
75 | return res.text();
76 | }
77 |
78 | function parseHitsFromHtml(html: string, limit = 20): BestMatchHit[] {
79 | const results: BestMatchHit[] = [];
80 |
81 | // Find all message wrapper divs with data-lia-message-uid
82 | const wrapperRegex = /<div[^>]+data-lia-message-uid="([^"]*)"[^>]*class="[^"]*lia-message-view-wrapper[^"]*"[^>]*>([\s\S]*?)(?=<div[^>]+class="[^"]*lia-message-view-wrapper|$)/gi;
83 | let match;
84 |
85 | while ((match = wrapperRegex.exec(html)) !== null && results.length < limit) {
86 | const postId = match[1];
87 | const seg = match[2].slice(0, 60000); // safety cap
88 |
89 | // Title + URL
90 | const titleMatch =
91 | seg.match(
92 | /<h2[^>]*class="[^"]*message-subject[^"]*"[^>]*>[\s\S]*?<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i
93 | ) ||
94 | seg.match(
95 | /<a[^>]+class="page-link[^"]*"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/i
96 | );
97 |
98 | const url = titleMatch ? absolutize(decodeEntities(titleMatch[1])) : "";
99 | const title = titleMatch ? stripTags(titleMatch[2]) : "";
100 | if (!title || !url) continue;
101 |
102 | // Author
103 | // Look for "View Profile of ..." or the user link block
104 | let author = "";
105 | const authorMatch =
106 | seg.match(/viewprofilepage\/user-id\/\d+[^>]*>([^<]+)/i) ||
107 | seg.match(/class="[^"]*lia-user-name-link[^"]*"[^>]*>([^<]+)/i);
108 | if (authorMatch) author = stripTags(authorMatch[1]);
109 |
110 | // Date/time
111 | const dateMatch = seg.match(/class="local-date"[^>]*>([^<]+)</i);
112 | const timeMatch = seg.match(/class="local-time"[^>]*>([^<]+)</i);
113 | const published = dateMatch
114 | ? `${stripTags(dateMatch[1])}${timeMatch ? " " + stripTags(timeMatch[1]) : ""}`
115 | : undefined;
116 |
117 | // Likes (Kudos)
118 | const likesMatch = seg.match(/Kudos Count\s+(\d+)/i);
119 | const likes = likesMatch ? Number(likesMatch[1]) : undefined;
120 |
121 | // Snippet
122 | const snippetMatch = seg.match(
123 | /<div[^>]*class="[^"]*lia-truncated-body-container[^"]*"[^>]*>([\s\S]*?)<\/div>/i
124 | );
125 | const snippet = snippetMatch ? stripTags(snippetMatch[1]).slice(0, CONFIG.EXCERPT_LENGTH_COMMUNITY) : undefined;
126 |
127 | // Tags
128 | const tagSectionMatch = seg.match(
129 | /<div[^>]*class="[^"]*TagList[^"]*"[^>]*>[\s\S]*?<\/div>/i
130 | );
131 | const tags: string[] = [];
132 | if (tagSectionMatch) {
133 | const tagLinks = tagSectionMatch[0].matchAll(
134 | /<a[^>]*class="[^"]*lia-tag[^"]*"[^>]*>([\s\S]*?)<\/a>/gi
135 | );
136 | for (const m of tagLinks) {
137 | const t = stripTags(m[1]);
138 | if (t) tags.push(t);
139 | }
140 | }
141 |
142 | results.push({ title, url, author, published, likes, snippet, tags, postId });
143 | }
144 |
145 | return results;
146 | }
147 |
148 | export async function searchCommunityBestMatch(
149 | query: string,
150 | opts: Options = {}
151 | ): Promise<BestMatchHit[]> {
152 | const { includeBlogs = true, limit = 20, userAgent } = opts;
153 | const url = buildSearchUrl(query, includeBlogs);
154 | const html = await fetchText(url, userAgent);
155 | return parseHitsFromHtml(html, limit);
156 | }
157 |
158 | // Convenience function: Search and get full content of top N posts in one call
159 | export async function searchAndGetTopPosts(
160 | query: string,
161 | topN: number = 3,
162 | opts: Options = {}
163 | ): Promise<{ search: BestMatchHit[], posts: { [id: string]: string } }> {
164 | // First, search for posts
165 | const searchResults = await searchCommunityBestMatch(query, { ...opts, limit: Math.max(topN, opts.limit || 20) });
166 |
167 | // Extract post IDs from top N results
168 | const topResults = searchResults.slice(0, topN);
169 | const postIds = topResults
170 | .map(result => result.postId)
171 | .filter((id): id is string => id !== undefined);
172 |
173 | // Batch retrieve full content
174 | const posts = await getCommunityPostsByIds(postIds, opts.userAgent);
175 |
176 | return {
177 | search: topResults,
178 | posts
179 | };
180 | }
181 |
182 | // Function to get full post content by scraping the post page
183 | // Batch retrieve multiple posts using LiQL API
184 | export async function getCommunityPostsByIds(postIds: string[], userAgent?: string): Promise<{ [id: string]: string }> {
185 | const results: { [id: string]: string } = {};
186 |
187 | if (postIds.length === 0) {
188 | return results;
189 | }
190 |
191 | try {
192 | // Build LiQL query for batch retrieval
193 | const idList = postIds.map(id => `'${id}'`).join(', ');
194 | const liqlQuery = `
195 | select body, id, subject, search_snippet, post_time, view_href
196 | from messages
197 | where id in (${idList})
198 | `.replace(/\s+/g, ' ').trim();
199 |
200 | const url = `https://community.sap.com/api/2.0/search?q=${encodeURIComponent(liqlQuery)}`;
201 |
202 | const response = await fetch(url, {
203 | headers: {
204 | 'Accept': 'application/json',
205 | 'User-Agent': userAgent || 'sap-docs-mcp/1.0 (BatchRetrieval)'
206 | }
207 | });
208 |
209 | if (!response.ok) {
210 | console.warn(`SAP Community API returned ${response.status}: ${response.statusText}`);
211 | return results;
212 | }
213 |
214 | const data = await response.json() as any;
215 |
216 | if (data.status !== 'success' || !data.data?.items) {
217 | return results;
218 | }
219 |
220 | // Process each post
221 | for (const post of data.data.items) {
222 | const postDate = post.post_time ? new Date(post.post_time).toLocaleDateString() : 'Unknown';
223 | const postUrl = post.view_href || `https://community.sap.com/t5/technology-blogs-by-sap/bg-p/t/${post.id}`;
224 |
225 | const fullContent = `# ${post.subject}
226 |
227 | **Source**: SAP Community Blog Post
228 | **Published**: ${postDate}
229 | **URL**: ${postUrl}
230 |
231 | ---
232 |
233 | ${post.body || post.search_snippet}
234 |
235 | ---
236 |
237 | *This content is from the SAP Community and represents community knowledge and experiences.*`;
238 |
239 | // Apply intelligent truncation if content is too large
240 | const truncationResult = truncateContent(fullContent);
241 | results[post.id] = truncationResult.content;
242 | }
243 |
244 | return results;
245 | } catch (error) {
246 | console.warn('Failed to batch retrieve community posts:', error);
247 | return results;
248 | }
249 | }
250 |
251 | // Single post retrieval using LiQL API
252 | export async function getCommunityPostById(postId: string, userAgent?: string): Promise<string | null> {
253 | const results = await getCommunityPostsByIds([postId], userAgent);
254 | return results[postId] || null;
255 | }
256 |
257 | export async function getCommunityPostByUrl(postUrl: string, userAgent?: string): Promise<string | null> {
258 | try {
259 | const html = await fetchText(postUrl, userAgent);
260 |
261 | // Extract title - try multiple selectors
262 | let title = "Untitled";
263 | const titleSelectors = [
264 | /<h1[^>]*class="[^"]*lia-message-subject[^"]*"[^>]*>([\s\S]*?)<\/h1>/i,
265 | /<h2[^>]*class="[^"]*message-subject[^"]*"[^>]*>([\s\S]*?)<\/h2>/i,
266 | /<title>([\s\S]*?)<\/title>/i
267 | ];
268 |
269 | for (const selector of titleSelectors) {
270 | const titleMatch = html.match(selector);
271 | if (titleMatch) {
272 | title = stripTags(titleMatch[1]).replace(/\s*-\s*SAP Community.*$/, '').trim();
273 | break;
274 | }
275 | }
276 |
277 | // Extract author and date - multiple patterns
278 | let author = "Unknown";
279 | const authorSelectors = [
280 | /class="[^"]*lia-user-name-link[^"]*"[^>]*>([^<]+)/i,
281 | /viewprofilepage\/user-id\/\d+[^>]*>([^<]+)/i,
282 | /"author"[^>]*>[\s\S]*?<[^>]*>([^<]+)/i
283 | ];
284 |
285 | for (const selector of authorSelectors) {
286 | const authorMatch = html.match(selector);
287 | if (authorMatch) {
288 | author = stripTags(authorMatch[1]);
289 | break;
290 | }
291 | }
292 |
293 | // Extract date and time
294 | const dateMatch = html.match(/class="local-date"[^>]*>([^<]+)</i);
295 | const timeMatch = html.match(/class="local-time"[^>]*>([^<]+)</i);
296 | const published = dateMatch
297 | ? `${stripTags(dateMatch[1])}${timeMatch ? " " + stripTags(timeMatch[1]) : ""}`
298 | : "Unknown";
299 |
300 | // Extract main content - try multiple content selectors
301 | let content = "Content not available";
302 | const contentSelectors = [
303 | /<div[^>]*class="[^"]*lia-message-body[^"]*"[^>]*>([\s\S]*?)<\/div>/i,
304 | /<div[^>]*class="[^"]*lia-message-body-content[^"]*"[^>]*>([\s\S]*?)<\/div>/i,
305 | /<div[^>]*class="[^"]*messageBody[^"]*"[^>]*>([\s\S]*?)<\/div>/i
306 | ];
307 |
308 | for (const selector of contentSelectors) {
309 | const contentMatch = html.match(selector);
310 | if (contentMatch) {
311 | // Clean up the content - remove script tags, preserve some formatting
312 | let rawContent = contentMatch[1]
313 | .replace(/<script[\s\S]*?<\/script>/gi, '')
314 | .replace(/<style[\s\S]*?<\/style>/gi, '')
315 | .replace(/<iframe[\s\S]*?<\/iframe>/gi, '[Embedded Content]');
316 |
317 | // Convert some HTML elements to markdown-like format
318 | rawContent = rawContent
319 | .replace(/<h([1-6])[^>]*>([\s\S]*?)<\/h[1-6]>/gi, (_, level, text) => {
320 | const hashes = '#'.repeat(parseInt(level) + 1);
321 | return `\n${hashes} ${stripTags(text)}\n`;
322 | })
323 | .replace(/<p[^>]*>([\s\S]*?)<\/p>/gi, '\n$1\n')
324 | .replace(/<br\s*\/?>/gi, '\n')
325 | .replace(/<strong[^>]*>([\s\S]*?)<\/strong>/gi, '**$1**')
326 | .replace(/<em[^>]*>([\s\S]*?)<\/em>/gi, '*$1*')
327 | .replace(/<code[^>]*>([\s\S]*?)<\/code>/gi, '`$1`')
328 | .replace(/<pre[^>]*>([\s\S]*?)<\/pre>/gi, '\n```\n$1\n```\n')
329 | .replace(/<ul[^>]*>([\s\S]*?)<\/ul>/gi, '$1')
330 | .replace(/<li[^>]*>([\s\S]*?)<\/li>/gi, '- $1\n');
331 |
332 | content = stripTags(rawContent).replace(/\n\s*\n\s*\n/g, '\n\n').trim();
333 | break;
334 | }
335 | }
336 |
337 | // Extract tags
338 | const tagSectionMatch = html.match(
339 | /<div[^>]*class="[^"]*TagList[^"]*"[^>]*>[\s\S]*?<\/div>/i
340 | );
341 | const tags: string[] = [];
342 | if (tagSectionMatch) {
343 | const tagLinks = tagSectionMatch[0].matchAll(
344 | /<a[^>]*class="[^"]*lia-tag[^"]*"[^>]*>([\s\S]*?)<\/a>/gi
345 | );
346 | for (const m of tagLinks) {
347 | const t = stripTags(m[1]);
348 | if (t) tags.push(t);
349 | }
350 | }
351 |
352 | // Extract kudos count
353 | let kudos = 0;
354 | const kudosMatch = html.match(/(\d+)\s+Kudos?/i);
355 | if (kudosMatch) {
356 | kudos = parseInt(kudosMatch[1]);
357 | }
358 |
359 | const tagsText = tags.length > 0 ? `\n**Tags:** ${tags.join(", ")}` : "";
360 | const kudosText = kudos > 0 ? `\n**Kudos:** ${kudos}` : "";
361 |
362 | const fullContent = `# ${title}
363 |
364 | **Source**: SAP Community Blog Post
365 | **Author**: ${author}
366 | **Published**: ${published}${kudosText}${tagsText}
367 | **URL**: ${postUrl}
368 |
369 | ---
370 |
371 | ${content}
372 |
373 | ---
374 |
375 | *This content is from the SAP Community and represents community knowledge and experiences.*`;
376 |
377 | // Apply intelligent truncation if content is too large
378 | const truncationResult = truncateContent(fullContent);
379 | return truncationResult.content;
380 | } catch (error) {
381 | console.warn('Failed to get community post:', error);
382 | return null;
383 | }
384 | }
```
--------------------------------------------------------------------------------
/scripts/summarize-src.js:
--------------------------------------------------------------------------------
```javascript
1 | #!/usr/bin/env node
2 |
3 | import fs from 'fs/promises';
4 | import path from 'path';
5 | import { fileURLToPath } from 'url';
6 |
7 | const __filename = fileURLToPath(import.meta.url);
8 | const __dirname = path.dirname(__filename);
9 |
10 | // Configuration
11 | const SRC_DIR = path.join(__dirname, '..', 'src');
12 | const TEST_DIR = path.join(__dirname, '..', 'test');
13 | const SRC_OUTPUT_FILE = path.join(__dirname, '..', 'src_context.txt');
14 | const TEST_OUTPUT_FILE = path.join(__dirname, '..', 'test_context.txt');
15 |
16 | // Content inclusion settings
17 | const INCLUDE_CONTENT = true;
18 | const MAX_CONTENT_SIZE = 50000; // Max characters per file to include
19 | const CONTENT_PREVIEW_SIZE = 500; // Characters for preview when file is too large
20 |
21 | // File type patterns
22 | const FILE_PATTERNS = {
23 | typescript: /\.(ts|tsx)$/,
24 | javascript: /\.(js|jsx)$/,
25 | json: /\.json$/,
26 | markdown: /\.(md|mdx)$/,
27 | yaml: /\.(yml|yaml)$/,
28 | xml: /\.xml$/,
29 | html: /\.html$/,
30 | css: /\.css$/,
31 | scss: /\.scss$/,
32 | sql: /\.sql$/,
33 | config: /\.(config|conf)$/
34 | };
35 |
36 | // Function to get file stats
37 | async function getFileStats(filePath) {
38 | try {
39 | const stats = await fs.stat(filePath);
40 | return {
41 | size: stats.size,
42 | created: stats.birthtime,
43 | modified: stats.mtime,
44 | isDirectory: stats.isDirectory()
45 | };
46 | } catch (error) {
47 | return null;
48 | }
49 | }
50 |
51 | // Function to get file type
52 | function getFileType(filename) {
53 | for (const [type, pattern] of Object.entries(FILE_PATTERNS)) {
54 | if (pattern.test(filename)) {
55 | return type;
56 | }
57 | }
58 | return 'other';
59 | }
60 |
61 | // Function to count lines in a file
62 | async function countLines(filePath) {
63 | try {
64 | const content = await fs.readFile(filePath, 'utf-8');
65 | return content.split('\n').length;
66 | } catch (error) {
67 | return 0;
68 | }
69 | }
70 |
71 | // Function to extract imports from TypeScript/JavaScript files
72 | async function extractImports(filePath) {
73 | try {
74 | const content = await fs.readFile(filePath, 'utf-8');
75 | const importRegex = /import\s+(?:(?:\{[^}]*\}|\*\s+as\s+\w+|\w+)(?:\s*,\s*(?:\{[^}]*\}|\*\s+as\s+\w+|\w+))*\s+from\s+)?['"`]([^'"`]+)['"`]/g;
76 | const imports = [];
77 | let match;
78 |
79 | while ((match = importRegex.exec(content)) !== null) {
80 | imports.push(match[1]);
81 | }
82 |
83 | return imports;
84 | } catch (error) {
85 | return [];
86 | }
87 | }
88 |
89 | // Function to extract exports from TypeScript/JavaScript files
90 | async function extractExports(filePath) {
91 | try {
92 | const content = await fs.readFile(filePath, 'utf-8');
93 | const exports = [];
94 |
95 | // Named exports: export { name1, name2 }
96 | const namedExportRegex = /export\s*\{\s*([^}]+)\s*\}/g;
97 | let match;
98 | while ((match = namedExportRegex.exec(content)) !== null) {
99 | const names = match[1].split(',').map(n => n.trim().split(' as ')[0].trim());
100 | exports.push(...names);
101 | }
102 |
103 | // Function/class exports: export function name() {} or export class Name {}
104 | const functionClassRegex = /export\s+(?:async\s+)?(?:function|class)\s+(\w+)/g;
105 | while ((match = functionClassRegex.exec(content)) !== null) {
106 | exports.push(match[1]);
107 | }
108 |
109 | // Variable exports: export const name = ...
110 | const variableRegex = /export\s+(?:const|let|var)\s+(\w+)/g;
111 | while ((match = variableRegex.exec(content)) !== null) {
112 | exports.push(match[1]);
113 | }
114 |
115 | // Default export
116 | if (content.includes('export default')) {
117 | exports.push('default');
118 | }
119 |
120 | return [...new Set(exports)]; // Remove duplicates
121 | } catch (error) {
122 | return [];
123 | }
124 | }
125 |
126 | // Function to read file content with size limit
127 | async function getFileContent(filePath, maxSize = MAX_CONTENT_SIZE) {
128 | try {
129 | const content = await fs.readFile(filePath, 'utf-8');
130 |
131 | if (content.length <= maxSize) {
132 | return {
133 | content,
134 | truncated: false,
135 | originalSize: content.length
136 | };
137 | } else {
138 | return {
139 | content: content.substring(0, CONTENT_PREVIEW_SIZE) + '\n\n... [Content truncated - file too large] ...\n\n' + content.substring(content.length - CONTENT_PREVIEW_SIZE),
140 | truncated: true,
141 | originalSize: content.length
142 | };
143 | }
144 | } catch (error) {
145 | return {
146 | content: `[Error reading file: ${error.message}]`,
147 | truncated: false,
148 | originalSize: 0
149 | };
150 | }
151 | }
152 |
153 | // Function to extract metadata from file content
154 | function extractMetadata(content, fileType, filePath) {
155 | const metadata = {
156 | functions: [],
157 | classes: [],
158 | interfaces: [],
159 | types: [],
160 | constants: [],
161 | comments: []
162 | };
163 |
164 | if (fileType === 'typescript' || fileType === 'javascript') {
165 | // Extract functions
166 | const functionRegex = /(?:export\s+)?(?:async\s+)?function\s+(\w+)/g;
167 | let match;
168 | while ((match = functionRegex.exec(content)) !== null) {
169 | metadata.functions.push(match[1]);
170 | }
171 |
172 | // Extract arrow functions
173 | const arrowFunctionRegex = /(?:export\s+)?(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?\([^)]*\)\s*=>/g;
174 | while ((match = arrowFunctionRegex.exec(content)) !== null) {
175 | metadata.functions.push(match[1]);
176 | }
177 |
178 | // Extract classes
179 | const classRegex = /(?:export\s+)?class\s+(\w+)/g;
180 | while ((match = classRegex.exec(content)) !== null) {
181 | metadata.classes.push(match[1]);
182 | }
183 |
184 | // Extract interfaces (TypeScript)
185 | if (fileType === 'typescript') {
186 | const interfaceRegex = /(?:export\s+)?interface\s+(\w+)/g;
187 | while ((match = interfaceRegex.exec(content)) !== null) {
188 | metadata.interfaces.push(match[1]);
189 | }
190 |
191 | // Extract type aliases
192 | const typeRegex = /(?:export\s+)?type\s+(\w+)/g;
193 | while ((match = typeRegex.exec(content)) !== null) {
194 | metadata.types.push(match[1]);
195 | }
196 | }
197 |
198 | // Extract constants
199 | const constRegex = /(?:export\s+)?const\s+([A-Z_][A-Z0-9_]*)\s*=/g;
200 | while ((match = constRegex.exec(content)) !== null) {
201 | metadata.constants.push(match[1]);
202 | }
203 |
204 | // Extract JSDoc comments
205 | const jsdocRegex = /\/\*\*[\s\S]*?\*\//g;
206 | const jsdocMatches = content.match(jsdocRegex);
207 | if (jsdocMatches) {
208 | metadata.comments = jsdocMatches.slice(0, 3); // First 3 JSDoc comments
209 | }
210 | }
211 |
212 | return metadata;
213 | }
214 |
215 | // Function to analyze directory recursively
216 | async function analyzeDirectory(dirPath, relativePath = '') {
217 | const items = await fs.readdir(dirPath);
218 | const analysis = {
219 | files: [],
220 | directories: [],
221 | totalFiles: 0,
222 | totalLines: 0,
223 | fileTypes: {},
224 | imports: new Set(),
225 | largestFiles: [],
226 | recentFiles: []
227 | };
228 |
229 | for (const item of items) {
230 | const fullPath = path.join(dirPath, item);
231 | const itemRelativePath = path.join(relativePath, item);
232 | const stats = await getFileStats(fullPath);
233 |
234 | if (!stats) continue;
235 |
236 | if (stats.isDirectory) {
237 | analysis.directories.push({
238 | name: item,
239 | path: itemRelativePath,
240 | stats
241 | });
242 |
243 | // Recursively analyze subdirectory
244 | const subAnalysis = await analyzeDirectory(fullPath, itemRelativePath);
245 | analysis.files.push(...subAnalysis.files);
246 | analysis.totalFiles += subAnalysis.totalFiles;
247 | analysis.totalLines += subAnalysis.totalLines;
248 | analysis.imports = new Set([...analysis.imports, ...subAnalysis.imports]);
249 |
250 | // Merge file types
251 | for (const [type, count] of Object.entries(subAnalysis.fileTypes)) {
252 | analysis.fileTypes[type] = (analysis.fileTypes[type] || 0) + count;
253 | }
254 | } else {
255 | const fileType = getFileType(item);
256 | const lines = await countLines(fullPath);
257 | const imports = fileType === 'typescript' || fileType === 'javascript'
258 | ? await extractImports(fullPath)
259 | : [];
260 | const exports = fileType === 'typescript' || fileType === 'javascript'
261 | ? await extractExports(fullPath)
262 | : [];
263 |
264 | // Get file content if enabled
265 | const fileContent = INCLUDE_CONTENT ? await getFileContent(fullPath) : null;
266 | const metadata = fileContent ? extractMetadata(fileContent.content, fileType, fullPath) : null;
267 |
268 | const fileInfo = {
269 | name: item,
270 | path: itemRelativePath,
271 | type: fileType,
272 | size: stats.size,
273 | lines,
274 | created: stats.created,
275 | modified: stats.modified,
276 | imports,
277 | exports,
278 | content: fileContent,
279 | metadata
280 | };
281 |
282 | analysis.files.push(fileInfo);
283 | analysis.totalFiles++;
284 | analysis.totalLines += lines;
285 | analysis.fileTypes[fileType] = (analysis.fileTypes[fileType] || 0) + 1;
286 |
287 | // Add imports to global set
288 | imports.forEach(imp => analysis.imports.add(imp));
289 | }
290 | }
291 |
292 | return analysis;
293 | }
294 |
295 | // Function to format file size
296 | function formatFileSize(bytes) {
297 | const sizes = ['B', 'KB', 'MB', 'GB'];
298 | if (bytes === 0) return '0 B';
299 | const i = Math.floor(Math.log(bytes) / Math.log(1024));
300 | return Math.round(bytes / Math.pow(1024, i) * 100) / 100 + ' ' + sizes[i];
301 | }
302 |
303 | // Function to format date
304 | function formatDate(date) {
305 | return date.toISOString().split('T')[0];
306 | }
307 |
308 | // Main function to analyze a specific directory
309 | async function generateSummaryForDirectory(dirPath, outputFile, dirName) {
310 | console.log(`🔍 Analyzing ${dirName} folder...`);
311 |
312 | try {
313 | // Check if directory exists
314 | const dirExists = await fs.access(dirPath).then(() => true).catch(() => false);
315 | if (!dirExists) {
316 | throw new Error(`Directory not found: ${dirPath}`);
317 | }
318 |
319 | // Analyze the entire directory
320 | const analysis = await analyzeDirectory(dirPath);
321 |
322 | // Sort files by size and modification date
323 | analysis.largestFiles = analysis.files
324 | .sort((a, b) => b.size - a.size)
325 | .slice(0, 10);
326 |
327 | analysis.recentFiles = analysis.files
328 | .sort((a, b) => b.modified - a.modified)
329 | .slice(0, 10);
330 |
331 | // Generate summary content
332 | const summary = generateSummaryContent(analysis, dirName);
333 |
334 | // Write to file
335 | await fs.writeFile(outputFile, summary, 'utf-8');
336 |
337 | console.log(`✅ Summary written to: ${outputFile}`);
338 | console.log(`📊 Total files: ${analysis.totalFiles}`);
339 | console.log(`📝 Total lines: ${analysis.totalLines.toLocaleString()}`);
340 | console.log(`📁 Directories: ${analysis.directories.length}`);
341 |
342 | return analysis;
343 |
344 | } catch (error) {
345 | console.error(`❌ Error generating ${dirName} summary:`, error.message);
346 | throw error;
347 | }
348 | }
349 |
350 | // Main function
351 | async function generateSummary() {
352 | try {
353 | // Analyze src directory
354 | await generateSummaryForDirectory(SRC_DIR, SRC_OUTPUT_FILE, 'src');
355 | console.log('');
356 |
357 | // Analyze test directory
358 | await generateSummaryForDirectory(TEST_DIR, TEST_OUTPUT_FILE, 'test');
359 |
360 | console.log('\n🎉 Both summaries generated successfully!');
361 |
362 | } catch (error) {
363 | console.error('❌ Error generating summaries:', error.message);
364 | process.exit(1);
365 | }
366 | }
367 |
368 | // Function to generate summary content
369 | function generateSummaryContent(analysis, dirName = 'src') {
370 | const now = new Date();
371 | const isTestDir = dirName === 'test';
372 |
373 | let content = `# ${isTestDir ? 'Test Code' : 'Source Code'} Analysis Summary
374 | Generated: ${now.toISOString()}
375 | Project: SAP Docs MCP
376 | ${isTestDir ? 'Test' : 'Source'} Directory: ${dirName}/
377 |
378 | ## 📊 Overview
379 | - Total Files: ${analysis.totalFiles.toLocaleString()}
380 | - Total Lines of Code: ${analysis.totalLines.toLocaleString()}
381 | - Directories: ${analysis.directories.length}
382 | - Unique Imports: ${analysis.imports.size}
383 |
384 | ## 📁 Directory Structure
385 | ${analysis.directories.map(dir => `- ${dir.path}/`).join('\n')}
386 |
387 | ## 📄 File Types Distribution
388 | ${Object.entries(analysis.fileTypes)
389 | .sort(([,a], [,b]) => b - a)
390 | .map(([type, count]) => `- ${type}: ${count} files`)
391 | .join('\n')}
392 |
393 | ## 🔝 Largest Files (by size)
394 | ${analysis.largestFiles.map((file, index) =>
395 | `${index + 1}. ${file.path} (${formatFileSize(file.size)}, ${file.lines} lines)`
396 | ).join('\n')}
397 |
398 | ## ⏰ Recently Modified Files
399 | ${analysis.recentFiles.map((file, index) =>
400 | `${index + 1}. ${file.path} (${formatDate(file.modified)})`
401 | ).join('\n')}
402 |
403 | ## 📋 Detailed File Analysis
404 | ${analysis.files
405 | .sort((a, b) => a.path.localeCompare(b.path))
406 | .map(file => {
407 | let fileSection = `### 📄 ${file.path}
408 | **Type:** ${file.type}
409 | **Size:** ${formatFileSize(file.size)}
410 | **Lines:** ${file.lines}
411 | **Modified:** ${formatDate(file.modified)}`;
412 |
413 | if (file.imports.length > 0) {
414 | fileSection += `\n**Imports:** ${file.imports.join(', ')}`;
415 | }
416 |
417 | if (file.exports.length > 0) {
418 | fileSection += `\n**Exports:** ${file.exports.join(', ')}`;
419 | }
420 |
421 | if (file.metadata) {
422 | const meta = file.metadata;
423 | if (meta.functions.length > 0) {
424 | fileSection += `\n**Functions:** ${meta.functions.join(', ')}`;
425 | }
426 | if (meta.classes.length > 0) {
427 | fileSection += `\n**Classes:** ${meta.classes.join(', ')}`;
428 | }
429 | if (meta.interfaces.length > 0) {
430 | fileSection += `\n**Interfaces:** ${meta.interfaces.join(', ')}`;
431 | }
432 | if (meta.types.length > 0) {
433 | fileSection += `\n**Types:** ${meta.types.join(', ')}`;
434 | }
435 | if (meta.constants.length > 0) {
436 | fileSection += `\n**Constants:** ${meta.constants.join(', ')}`;
437 | }
438 | }
439 |
440 | if (file.content && INCLUDE_CONTENT) {
441 | fileSection += `\n\n**Content:**`;
442 | if (file.content.truncated) {
443 | fileSection += ` (${formatFileSize(file.content.originalSize)} - truncated)`;
444 | }
445 | fileSection += `\n\`\`\`${file.type === 'typescript' ? 'typescript' : file.type === 'javascript' ? 'javascript' : ''}\n${file.content.content}\n\`\`\``;
446 | }
447 |
448 | return fileSection;
449 | })
450 | .join('\n\n')}
451 |
452 | ## 🔗 Most Common Imports
453 | ${Array.from(analysis.imports)
454 | .sort()
455 | .slice(0, 20)
456 | .map(imp => `- ${imp}`)
457 | .join('\n')}
458 |
459 | ## 🔍 Code Analysis Summary
460 | ${(() => {
461 | const allFunctions = analysis.files.flatMap(f => f.metadata?.functions || []);
462 | const allClasses = analysis.files.flatMap(f => f.metadata?.classes || []);
463 | const allInterfaces = analysis.files.flatMap(f => f.metadata?.interfaces || []);
464 | const allTypes = analysis.files.flatMap(f => f.metadata?.types || []);
465 | const allExports = analysis.files.flatMap(f => f.exports || []);
466 |
467 | return `- Total Functions: ${allFunctions.length}
468 | - Total Classes: ${allClasses.length}
469 | - Total Interfaces: ${allInterfaces.length}
470 | - Total Types: ${allTypes.length}
471 | - Total Exports: ${allExports.length}
472 | - Files with Content: ${analysis.files.filter(f => f.content).length}`;
473 | })()}
474 |
475 | ## 📈 Statistics
476 | - Average file size: ${formatFileSize(analysis.files.reduce((sum, f) => sum + f.size, 0) / analysis.totalFiles)}
477 | - Average lines per file: ${Math.round(analysis.totalLines / analysis.totalFiles)}
478 | - Most common file type: ${Object.entries(analysis.fileTypes).sort(([,a], [,b]) => b - a)[0]?.[0] || 'N/A'}
479 | - Oldest file: ${analysis.files.length > 0 ? formatDate(analysis.files.reduce((oldest, f) => f.created < oldest.created ? f : oldest).created) : 'N/A'}
480 | - Newest file: ${analysis.files.length > 0 ? formatDate(analysis.files.reduce((newest, f) => f.modified > newest.modified ? f : newest).modified) : 'N/A'}
481 | - Content included: ${INCLUDE_CONTENT ? 'Yes' : 'No'}
482 | - Max content size per file: ${formatFileSize(MAX_CONTENT_SIZE)}
483 |
484 | ---
485 | Generated by summarize-src.js for ${dirName}/ directory
486 | `;
487 |
488 | return content;
489 | }
490 |
491 | // Run the script
492 | generateSummary();
493 |
```
--------------------------------------------------------------------------------
/.github/workflows/update-submodules.yml:
--------------------------------------------------------------------------------
```yaml
1 | name: Documentation Update & Database Health Monitor
2 |
3 | on:
4 | # Run daily at 4 AM UTC for submodule updates
5 | schedule:
6 | - cron: '0 4 * * *'
7 | # Run every 2 hours for database health monitoring
8 | - cron: '0 */2 * * *'
9 |
10 | # Allow manual triggering with options
11 | workflow_dispatch:
12 | inputs:
13 | health_check_only:
14 | description: 'Perform health check only (no submodule update)'
15 | required: false
16 | default: 'false'
17 | type: boolean
18 | force_rebuild:
19 | description: 'Force database rebuild even if healthy'
20 | required: false
21 | default: 'false'
22 | type: boolean
23 |
24 | jobs:
25 | update-submodules:
26 | name: Update Submodules on Server
27 | runs-on: ubuntu-22.04
28 | environment:
29 | name: remove server
30 |
31 | steps:
32 | - name: Update submodules on server with database safety checks
33 | uses: appleboy/[email protected]
34 | with:
35 | host: ${{ secrets.SERVER_IP }}
36 | username: ${{ secrets.SERVER_USERNAME }}
37 | key: ${{ secrets.SSH_PRIVATE_KEY }}
38 | script: |
39 | set -e
40 | cd /opt/mcp-sap/mcp-sap-docs
41 |
42 | DB_PATH="/opt/mcp-sap/mcp-sap-docs/dist/data/docs.sqlite"
43 | HEALTH_CHECK_ONLY="${{ inputs.health_check_only }}"
44 | FORCE_REBUILD="${{ inputs.force_rebuild }}"
45 |
46 | # Determine if this is a health check or full update
47 | IS_HEALTH_CHECK_SCHEDULE=false
48 | if [ "$(date +%H)" != "04" ] && [ "$HEALTH_CHECK_ONLY" != "true" ]; then
49 | IS_HEALTH_CHECK_SCHEDULE=true
50 | echo "=== Database Health Check (Scheduled) Started ==="
51 | elif [ "$HEALTH_CHECK_ONLY" = "true" ]; then
52 | echo "=== Database Health Check (Manual) Started ==="
53 | else
54 | echo "=== Documentation Update with Database Safety Started ==="
55 | fi
56 | echo "📅 $(date)"
57 | echo "🔍 Health check only: $HEALTH_CHECK_ONLY"
58 | echo "🔧 Force rebuild: $FORCE_REBUILD"
59 |
60 | # Function to check SQLite database integrity
61 | check_db_integrity() {
62 | local db_path="$1"
63 | if [ -f "$db_path" ]; then
64 | echo "🔍 Checking database integrity: $db_path"
65 | if sqlite3 "$db_path" "PRAGMA integrity_check;" 2>/dev/null | grep -q "ok"; then
66 | echo "✅ Database integrity OK"
67 | return 0
68 | else
69 | echo "❌ Database corruption detected"
70 | return 1
71 | fi
72 | else
73 | echo "ℹ️ Database file does not exist: $db_path"
74 | return 1
75 | fi
76 | }
77 |
78 | echo "==> Database health check"
79 | DB_WAS_CORRUPT=0
80 | DB_NEEDS_REPAIR=0
81 |
82 | if ! check_db_integrity "$DB_PATH"; then
83 | DB_WAS_CORRUPT=1
84 | DB_NEEDS_REPAIR=1
85 | echo "⚠️ Database corruption detected"
86 | fi
87 |
88 | # Force rebuild if requested
89 | if [ "$FORCE_REBUILD" = "true" ]; then
90 | DB_NEEDS_REPAIR=1
91 | echo "🔄 Force rebuild requested"
92 | fi
93 |
94 | # For health checks, test search functionality
95 | if [ "$IS_HEALTH_CHECK_SCHEDULE" = "true" ] || [ "$HEALTH_CHECK_ONLY" = "true" ]; then
96 | echo "==> Testing search functionality"
97 | SEARCH_TEST=$(curl -s -X POST http://127.0.0.1:3001/mcp \
98 | -H "Content-Type: application/json" \
99 | -d '{"role": "user", "content": "health check search"}' || echo "curl_failed")
100 |
101 | if echo "$SEARCH_TEST" | grep -q "SqliteError\|SQLITE_CORRUPT\|Tool execution failed\|curl_failed"; then
102 | DB_NEEDS_REPAIR=1
103 | echo "❌ Search functionality failed - repair needed"
104 | echo "Response: $SEARCH_TEST"
105 | else
106 | echo "✅ Search functionality OK"
107 | fi
108 | fi
109 |
110 | # Only proceed with backup and rebuilds if needed
111 | REPAIR_PERFORMED=0
112 | SUBMODULES_UPDATED=0
113 |
114 | if [ "$DB_NEEDS_REPAIR" -eq 1 ]; then
115 | echo ""
116 | echo "🔧 DATABASE REPAIR REQUIRED"
117 | echo "=========================="
118 |
119 | echo "==> Creating backup before repair"
120 | BACKUP_DIR="/opt/mcp-sap/backups"
121 | mkdir -p "$BACKUP_DIR"
122 |
123 | if [ -f "$DB_PATH" ] && [ "$DB_WAS_CORRUPT" -eq 0 ]; then
124 | BACKUP_PATH="$BACKUP_DIR/pre-repair-$(date +%Y%m%d-%H%M%S).sqlite"
125 | cp "$DB_PATH" "$BACKUP_PATH"
126 | echo "✅ Database backed up to $BACKUP_PATH"
127 | fi
128 |
129 | echo "==> Checking system resources"
130 | # Check disk space
131 | AVAILABLE_MB=$(df /opt/mcp-sap --output=avail -m | tail -n1)
132 | if [ "$AVAILABLE_MB" -lt 1000 ]; then
133 | echo "❌ ERROR: Insufficient disk space. Available: ${AVAILABLE_MB}MB, Required: 1000MB"
134 | exit 1
135 | fi
136 | echo "✅ Disk space OK: ${AVAILABLE_MB}MB available"
137 |
138 | # Check memory
139 | AVAILABLE_KB=$(awk '/MemAvailable/ { print $2 }' /proc/meminfo)
140 | AVAILABLE_MB_MEM=$((AVAILABLE_KB / 1024))
141 | if [ "$AVAILABLE_MB_MEM" -lt 512 ]; then
142 | echo "❌ ERROR: Insufficient memory. Available: ${AVAILABLE_MB_MEM}MB, Required: 512MB"
143 | exit 1
144 | fi
145 | echo "✅ Memory OK: ${AVAILABLE_MB_MEM}MB available"
146 |
147 | echo "==> Gracefully stopping services for repair"
148 | pm2 stop all || true
149 | sleep 3
150 |
151 | # Remove corrupted database
152 | echo "🗑️ Removing corrupted database"
153 | rm -f "$DB_PATH"
154 |
155 | # Rebuild database
156 | echo "🔨 Rebuilding database..."
157 | npm run build:fts
158 |
159 | REPAIR_PERFORMED=1
160 |
161 | elif [ "$IS_HEALTH_CHECK_SCHEDULE" != "true" ] && [ "$HEALTH_CHECK_ONLY" != "true" ]; then
162 | echo ""
163 | echo "📦 SUBMODULE UPDATE PROCESS"
164 | echo "=========================="
165 |
166 | echo "==> Creating backup before update"
167 | BACKUP_DIR="/opt/mcp-sap/backups"
168 | mkdir -p "$BACKUP_DIR"
169 |
170 | if [ -f "$DB_PATH" ]; then
171 | BACKUP_PATH="$BACKUP_DIR/pre-update-$(date +%Y%m%d-%H%M%S).sqlite"
172 | cp "$DB_PATH" "$BACKUP_PATH"
173 | echo "✅ Database backed up to $BACKUP_PATH"
174 | fi
175 |
176 | echo "==> Checking system resources"
177 | # Check disk space
178 | AVAILABLE_MB=$(df /opt/mcp-sap --output=avail -m | tail -n1)
179 | if [ "$AVAILABLE_MB" -lt 500 ]; then
180 | echo "❌ ERROR: Insufficient disk space. Available: ${AVAILABLE_MB}MB, Required: 500MB"
181 | exit 1
182 | fi
183 | echo "✅ Disk space OK: ${AVAILABLE_MB}MB available"
184 |
185 | echo "==> Gracefully stopping services for update"
186 | pm2 stop all || true
187 | sleep 3
188 |
189 | # Configure git for HTTPS
190 | git config --global url."https://github.com/".insteadOf [email protected]:
191 |
192 | echo "==> Checking for submodule updates"
193 | # Get current submodule commits
194 | git submodule status > /tmp/before-update.txt || true
195 |
196 | # Reuse setup.sh to ensure shallow, single-branch submodules and build
197 | echo "==> Running setup script (includes submodule update and rebuild)"
198 | SKIP_NESTED_SUBMODULES=1 bash setup.sh
199 |
200 | # Check what changed
201 | git submodule status > /tmp/after-update.txt || true
202 |
203 | if ! diff -q /tmp/before-update.txt /tmp/after-update.txt >/dev/null 2>&1; then
204 | echo "✅ Submodules were updated - changes detected"
205 | SUBMODULES_UPDATED=1
206 | else
207 | echo "ℹ️ No submodule changes detected"
208 | SUBMODULES_UPDATED=0
209 | fi
210 | else
211 | echo "ℹ️ Health check only - no database repair or submodule update needed"
212 | fi
213 |
214 | # Post-operation database integrity check (only if changes were made)
215 | if [ "$REPAIR_PERFORMED" -eq 1 ] || [ "$SUBMODULES_UPDATED" -eq 1 ]; then
216 | echo "==> Post-operation database integrity check"
217 | if ! check_db_integrity "$DB_PATH"; then
218 | echo "❌ ERROR: Database corruption after operation"
219 |
220 | # Try to restore from backup if available
221 | if [ -f "$BACKUP_PATH" ] && [ "$DB_WAS_CORRUPT" -eq 0 ]; then
222 | echo "🔄 Attempting to restore from backup..."
223 | cp "$BACKUP_PATH" "$DB_PATH"
224 |
225 | if check_db_integrity "$DB_PATH"; then
226 | echo "✅ Successfully restored from backup"
227 | else
228 | echo "❌ Backup is also corrupted - forcing fresh rebuild"
229 | rm -f "$DB_PATH"
230 | npm run build:fts
231 | fi
232 | else
233 | echo "🔄 No clean backup available - forcing fresh rebuild"
234 | rm -f "$DB_PATH"
235 | npm run build:fts
236 | fi
237 |
238 | # Final check
239 | if ! check_db_integrity "$DB_PATH"; then
240 | echo "❌ CRITICAL: Could not create working database"
241 | exit 1
242 | fi
243 | fi
244 | fi
245 |
246 | # Restart services if they were stopped
247 | if [ "$REPAIR_PERFORMED" -eq 1 ] || [ "$SUBMODULES_UPDATED" -eq 1 ]; then
248 | echo "==> Creating logs directory"
249 | mkdir -p /opt/mcp-sap/logs
250 | chown -R "$USER":"$USER" /opt/mcp-sap/logs
251 |
252 | echo "==> Restarting services"
253 | pm2 start all
254 | pm2 save
255 | sleep 10
256 | fi
257 |
258 | echo "==> Health verification"
259 | # Test search functionality
260 | SEARCH_TEST=$(curl -s -X POST http://127.0.0.1:3001/mcp \
261 | -H "Content-Type: application/json" \
262 | -d '{"role": "user", "content": "test search"}' || echo "curl_failed")
263 |
264 | if echo "$SEARCH_TEST" | grep -q "SqliteError\|SQLITE_CORRUPT\|Tool execution failed\|curl_failed"; then
265 | echo "❌ ERROR: Search functionality test failed after update"
266 | echo "Response: $SEARCH_TEST"
267 | exit 1
268 | else
269 | echo "✅ Search functionality verified"
270 | fi
271 |
272 | echo "==> Cleanup old backups (keep last 7)"
273 | ls -t "$BACKUP_DIR"/*.sqlite 2>/dev/null | tail -n +8 | xargs -r rm --
274 |
275 | echo ""
276 | if [ "$IS_HEALTH_CHECK_SCHEDULE" = "true" ] || [ "$HEALTH_CHECK_ONLY" = "true" ]; then
277 | echo "=== Health Check Completed Successfully ==="
278 | echo "🔍 Database health: $([ "$DB_NEEDS_REPAIR" -eq 0 ] && echo "Healthy" || echo "Repaired")"
279 | echo "🔧 Repair performed: $([ "$REPAIR_PERFORMED" -eq 1 ] && echo "Yes" || echo "No")"
280 | else
281 | echo "=== Documentation Update Completed Successfully ==="
282 | echo "📊 Submodules updated: $([ "$SUBMODULES_UPDATED" -eq 1 ] && echo "Yes" || echo "No")"
283 | echo "🔧 Database repair: $([ "$REPAIR_PERFORMED" -eq 1 ] && echo "Yes" || echo "No")"
284 | fi
285 | echo "📅 Completion time: $(date)"
286 |
287 | - name: Create success summary
288 | if: success()
289 | run: |
290 | echo "## ✅ Documentation Update Complete (Enhanced with DB Safety)" >> $GITHUB_STEP_SUMMARY
291 | echo "" >> $GITHUB_STEP_SUMMARY
292 | echo "### Update Results:" >> $GITHUB_STEP_SUMMARY
293 | echo "✅ **Submodules**: Updated to latest remote commits" >> $GITHUB_STEP_SUMMARY
294 | echo "✅ **Search Index**: Rebuilt with enhanced safety checks" >> $GITHUB_STEP_SUMMARY
295 | echo "✅ **Database Health**: Verified before and after update" >> $GITHUB_STEP_SUMMARY
296 | echo "✅ **Services**: Restarted and verified working" >> $GITHUB_STEP_SUMMARY
297 | echo "✅ **Backup**: Pre-update backup created (if needed)" >> $GITHUB_STEP_SUMMARY
298 | echo "" >> $GITHUB_STEP_SUMMARY
299 | echo "### Safety Features Added:" >> $GITHUB_STEP_SUMMARY
300 | echo "- 🔍 **Database integrity checks** before and after update" >> $GITHUB_STEP_SUMMARY
301 | echo "- 📦 **Automatic backup** of healthy database before changes" >> $GITHUB_STEP_SUMMARY
302 | echo "- 🔧 **Automatic corruption recovery** with backup restoration" >> $GITHUB_STEP_SUMMARY
303 | echo "- 🧪 **Search functionality verification** after update" >> $GITHUB_STEP_SUMMARY
304 | echo "- 💾 **System resource validation** before rebuild" >> $GITHUB_STEP_SUMMARY
305 | echo "" >> $GITHUB_STEP_SUMMARY
306 | echo "🕐 **Update time**: $(date -u)" >> $GITHUB_STEP_SUMMARY
307 | echo "" >> $GITHUB_STEP_SUMMARY
308 | echo "The MCP server now has the latest SAP documentation with verified database integrity."
309 |
310 | - name: Create failure summary
311 | if: failure()
312 | run: |
313 | echo "## ❌ Documentation Update Failed" >> $GITHUB_STEP_SUMMARY
314 | echo "" >> $GITHUB_STEP_SUMMARY
315 | echo "🚨 **Status**: Enhanced update process encountered errors" >> $GITHUB_STEP_SUMMARY
316 | echo "" >> $GITHUB_STEP_SUMMARY
317 | echo "### Enhanced Troubleshooting:" >> $GITHUB_STEP_SUMMARY
318 | echo "**Check these areas on the server:**" >> $GITHUB_STEP_SUMMARY
319 | echo "1. **Database corruption**: Check for SQLite errors in logs" >> $GITHUB_STEP_SUMMARY
320 | echo "2. **System resources**: Disk space and memory availability" >> $GITHUB_STEP_SUMMARY
321 | echo "3. **Service status**: PM2 process health and logs" >> $GITHUB_STEP_SUMMARY
322 | echo "4. **Backup availability**: Check /opt/mcp-sap/backups/" >> $GITHUB_STEP_SUMMARY
323 | echo "" >> $GITHUB_STEP_SUMMARY
324 | echo "### Quick Recovery Commands:" >> $GITHUB_STEP_SUMMARY
325 | echo "\`\`\`bash" >> $GITHUB_STEP_SUMMARY
326 | echo "# Check PM2 status and logs" >> $GITHUB_STEP_SUMMARY
327 | echo "pm2 status && pm2 logs --lines 50" >> $GITHUB_STEP_SUMMARY
328 | echo "" >> $GITHUB_STEP_SUMMARY
329 | echo "# Manual database rebuild if corruption detected" >> $GITHUB_STEP_SUMMARY
330 | echo "cd /opt/mcp-sap/mcp-sap-docs" >> $GITHUB_STEP_SUMMARY
331 | echo "pm2 stop all" >> $GITHUB_STEP_SUMMARY
332 | echo "rm -f dist/data/docs.sqlite" >> $GITHUB_STEP_SUMMARY
333 | echo "npm run build:fts" >> $GITHUB_STEP_SUMMARY
334 | echo "pm2 start all" >> $GITHUB_STEP_SUMMARY
335 | echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
336 | echo "" >> $GITHUB_STEP_SUMMARY
337 | echo "🕐 **Failure time**: $(date -u)" >> $GITHUB_STEP_SUMMARY
```