┌─────────────────────────────────────────────┐
│  Input: Tool File with Product URL          │
│  - Reads frontmatter: url, site_name        │
│  - Optional: parent_org, existing links     │
└─────────────────────────────────────────────┘
              ↓
┌─────────────────────────────────────────────┐
│  Link Discovery Services (Parallel)         │
│  1. Website Scraper (social icons/links)    │
│  2. Search Engine Queries (Google, DDG)     │
│  3. Common URL Pattern Tester               │
│  4. Jina Reader (deep content extraction)   │
│  5. LLM Analysis (structured data extract)  │
│  6. Optional: Clearbit/FullContact APIs     │
└─────────────────────────────────────────────┘
              ↓
┌─────────────────────────────────────────────┐
│  Link Validator & Scorer                    │
│  - HTTP status check (200 OK)               │
│  - Confidence scoring per link              │
│  - False positive filtering                 │
└─────────────────────────────────────────────┘
              ↓
┌─────────────────────────────────────────────┐
│  Link Categorizer & Normalizer              │
│  - Classify link type (GitHub, LinkedIn)    │
│  - Normalize URLs (canonical forms)         │
│  - Extract key identifiers (usernames)      │
└─────────────────────────────────────────────┘
              ↓
┌─────────────────────────────────────────────┐
│  Metadata Writer                            │
│  - Updates tool frontmatter                 │
│  - Preserves existing manual links          │
│  - Adds confidence scores                   │
│  - Triggers Filesystem Observer             │
└─────────────────────────────────────────────┘

---
url: https://www.trae.ai/
site_name: Trae
parent_org: "[[organizations/ByteDance|ByteDance]]"
---

---
url: https://www.trae.ai/
site_name: Trae
parent_org: "[[organizations/ByteDance|ByteDance]]"
linkedin_url: https://www.linkedin.com/company/trae-ai/
twitter_url: https://twitter.com/traeai
bluesky_url: https://bsky.app/profile/trae.ai
youtube_channel_url: https://www.youtube.com/@traeai
github_org_url: https://github.com/traehq
github_repo_url: https://github.com/traehq/trae
medium_url: https://medium.com/@traeai
product_hunt_url: https://www.producthunt.com/products/trae
discord_url: https://discord.gg/traeai
reddit_url: https://www.reddit.com/r/traeai
blog_url: https://www.trae.ai/blog
blog_rss_url: https://www.trae.ai/blog/rss.xml
changelog_url: https://www.trae.ai/changelog
docs_url: https://docs.trae.ai
links_last_updated: 2025-11-15
links_auto_discovered: true
links_confidence: high
links_manually_verified: false
---

async function scrapeSocialLinks(url) {
  const page = await browser.newPage();
  await page.goto(url);

  // Common selectors for social links
  const socialSelectors = [
    'a[href*="linkedin.com/company"]',
    'a[href*="twitter.com"]',
    'a[href*="x.com"]',
    'a[href*="github.com"]',
    'a[href*="youtube.com/channel"]',
    'a[href*="youtube.com/@"]',
    'a[href*="medium.com/@"]',
    'a[href*="producthunt.com/products"]',
    // ... more selectors
  ];

  const links = await page.$$eval(
    socialSelectors.join(', '),
    anchors => anchors.map(a => a.href)
  );

  return links;
}

"Trae AI" site:linkedin.com/company
"Trae AI" site:github.com
"Trae AI" site:twitter.com OR site:x.com
"Trae AI" site:youtube.com
"Trae AI" site:medium.com
"Trae AI" site:producthunt.com/products

const commonPatterns = {
  rss: [
    '/rss.xml',
    '/feed',
    '/feed.xml',
    '/blog/rss.xml',
    '/blog/feed',
    '/atom.xml'
  ],
  changelog: [
    '/changelog',
    '/releases',
    '/release-notes',
    '/updates',
    '/whats-new'
  ],
  docs: [
    '/docs',
    '/documentation',
    '/developer',
    '/api-docs',
    'https://docs.{domain}'
  ],
  github: [
    'https://github.com/{company-name}',
    'https://github.com/{product-name}'
  ]
};

async function jinaExtractLinks(url) {
  const jinaUrl = `https://r.jina.ai/${url}`;
  const response = await fetch(jinaUrl);
  const markdown = await response.text();

  // Extract all markdown links: [text](url)
  const linkRegex = /\[([^\]]+)\]\(([^)]+)\)/g;
  const links = [];
  let match;

  while ((match = linkRegex.exec(markdown)) !== null) {
    links.push({
      text: match[1],
      url: match[2]
    });
  }

  return links;
}

Analyze this webpage content and extract social media profiles and important links.

Product: {product_name}
Homepage URL: {product_url}
Page Content:
{jina_content}

Extract the following if found:
1. LinkedIn company page URL
2. Twitter/X official account URL
3. GitHub organization or repository URL
4. YouTube channel URL
5. Blog RSS feed URL
6. Changelog/Release notes URL
7. Documentation URL
8. Any other official social media profiles

For each link found, provide:
- link_type: (linkedin | twitter | github | youtube | rss | changelog | docs | other)
- url: (full URL)
- confidence: (high | medium | low)
- reasoning: (why you think this is the official link)

Return as JSON array.

{
  "name": "Trae",
  "domain": "trae.ai",
  "twitter": {
    "handle": "traeai",
    "followers": 5234
  },
  "linkedin": {
    "handle": "company/trae-ai"
  },
  "github": {
    "handle": "traehq"
  },
  "tech": ["react", "node.js"],
  "description": "AI-powered development tools"
}

async function validateLink(url, expectedType) {
  try {
    // 1. HTTP Status Check
    const response = await fetch(url, { method: 'HEAD', redirect: 'follow' });
    if (response.status !== 200) return { valid: false, reason: 'HTTP error' };

    // 2. Type-Specific Validation
    if (expectedType === 'github') {
      // Check if it's actually a GitHub org/repo
      const isOrg = url.match(/github\.com\/[^/]+\/?$/);
      const isRepo = url.match(/github\.com\/[^/]+\/[^/]+\/?$/);
      if (!isOrg && !isRepo) return { valid: false, reason: 'Not a valid GitHub URL' };
    }

    if (expectedType === 'rss') {
      // Check if RSS feed is valid XML
      const content = await fetch(url).then(r => r.text());
      if (!content.includes('<rss') && !content.includes('<feed')) {
        return { valid: false, reason: 'Not a valid RSS feed' };
      }
    }

    // 3. Check for redirects to unrelated domains
    const finalUrl = response.url;
    const originalDomain = new URL(url).hostname;
    const finalDomain = new URL(finalUrl).hostname;

    if (!domainsRelated(originalDomain, finalDomain)) {
      return { valid: false, reason: 'Redirects to unrelated domain' };
    }

    return { valid: true };
  } catch (error) {
    return { valid: false, reason: error.message };
  }
}

function calculateConfidence(link, discoveryMethod, validationResult) {
  let score = 0;

  // Discovery method weight
  const methodScores = {
    'website_scraper': 30,      // Found on official site
    'common_pattern': 25,       // Standard URL pattern worked
    'jina_extraction': 20,      // Found in page content
    'search_engine': 15,        // Found via search
    'llm_analysis': 10,         // AI identified
    'clearbit_api': 35          // Verified by paid service
  };
  score += methodScores[discoveryMethod] || 0;

  // Validation checks
  if (validationResult.valid) score += 30;
  if (validationResult.httpsOnly) score += 10;
  if (validationResult.noRedirects) score += 10;

  // Context signals
  if (link.foundInMultiplePlaces) score += 20;
  if (link.matchesCompanyName) score += 10;
  if (link.verifiedBadge) score += 15;  // e.g., Twitter verified

  // Classify
  if (score >= 70) return 'high';
  if (score >= 40) return 'medium';
  return 'low';
}

const linkPatterns = {
  linkedin: /linkedin\.com\/company\/([^/?]+)/,
  twitter: /(twitter|x)\.com\/([^/?]+)/,
  bluesky: /bsky\.app\/profile\/([^/?]+)/,
  github_org: /github\.com\/([^/]+)\/?$/,
  github_repo: /github\.com\/([^/]+)\/([^/]+)/,
  youtube: /youtube\.com\/(channel\/[^/?]+|@[^/?]+)/,
  medium: /medium\.com\/@([^/?]+)/,
  product_hunt: /producthunt\.com\/products\/([^/?]+)/,
  discord: /(discord\.gg|discord\.com\/invite)\/([^/?]+)/,
  rss: /\/(rss|feed|atom)\.(xml|rss)$/
};

// .state/link-aggregator/discovery-cache.json
{
  "tools": {
    "trae-ai": {
      "url": "https://www.trae.ai/",
      "last_scanned": "2025-11-15T10:30:00Z",
      "links_found": 12,
      "links_validated": 10,
      "confidence": "high",
      "discovery_methods_used": [
        "website_scraper",
        "common_pattern",
        "jina_extraction"
      ],
      "links": {
        "linkedin_url": {
          "url": "https://www.linkedin.com/company/trae-ai/",
          "method": "website_scraper",
          "confidence": "high",
          "validated": true,
          "first_seen": "2025-11-15T10:30:00Z"
        },
        // ... more links
      }
    }
  }
}

// tidyverse/observers/userOptionsConfig.ts
export const linkAggregatorConfig = {
  enabled: true,
  runOnNewFiles: true,  // Auto-discover links for new tools
  runOnManualTrigger: true,
  confidenceThreshold: 'medium',  // Only add medium+ confidence
  requiredInputFields: ['url', 'site_name'],
  outputFields: [
    'linkedin_url',
    'twitter_url',
    'github_repo_url',
    'blog_rss_url',
    // ... more
  ]
};

---
watch_enabled: false
tool_ref: "tooling/AI-Toolkit/.../Trae AI.md"
sources:
  - type: github_releases
    repo: traehq/trae
  - type: rss
    url: https://www.trae.ai/blog/rss.xml
  - type: changelog_page
    url: https://www.trae.ai/changelog

# tooling/AI-Toolkit/.../Trae AI.md
parent_org: "[[organizations/ByteDance|ByteDance]]"

# Then also discover:
# - ByteDance LinkedIn
# - ByteDance GitHub
# - ByteDance careers page
# - etc.

// .state/link-aggregator/failed-tools.json
{
  "failed": [
    {
      "tool": "tooling/AI-Toolkit/.../Cursor.md",
      "reason": "Rate limited by website",
      "timestamp": "2025-11-15T10:30:00Z",
      "retry_count": 2,
      "next_retry": "2025-11-15T11:30:00Z"
    }
  ]
}

# .state/link-aggregator/review-queue.md

## Trae AI
**Confidence**: Medium
**Links to Review**:
- [ ] Twitter: https://twitter.com/trae_official (found via search, not website)
- [ ] Discord: https://discord.gg/traeai (found in footer, but 404)
- [ ] YouTube: https://youtube.com/@traeai (found via LLM, low confidence)

**Actions**:
- ✅ Approve
- ❌ Reject
- ✏️ Edit URL

---
links_manually_verified: true
links_manual_overrides:
  twitter_url: https://x.com/correct_handle
---

describe('Link Discovery', () => {
  test('scrapes social links from homepage', async () => {
    const links = await scrapeSocialLinks('https://www.trae.ai/');
    expect(links).toContain('https://github.com/traehq');
  });

  test('validates GitHub repo URL', async () => {
    const result = await validateLink('https://github.com/traehq/trae', 'github');
    expect(result.valid).toBe(true);
  });

  test('calculates confidence score correctly', () => {
    const confidence = calculateConfidence({
      discoveryMethod: 'website_scraper',
      validationResult: { valid: true },
      foundInMultiplePlaces: true
    });
    expect(confidence).toBe('high');
  });
});

describe('End-to-End Link Aggregation', () => {
  test('discovers and validates links for Trae AI', async () => {
    const tool = {
      url: 'https://www.trae.ai/',
      site_name: 'Trae'
    };

    const result = await discoverLinks(tool);

    expect(result.linkedin_url).toBeDefined();
    expect(result.github_repo_url).toBeDefined();
    expect(result.links_confidence).toBe('high');
    expect(result.links_found).toBeGreaterThan(5);
  });
});

tidyverse/link-aggregator/
├── src/
│   ├── index.ts                    # Main orchestrator
│   ├── config/
│   │   └── patterns.ts             # URL patterns, selectors
│   ├── discoverers/
│   │   ├── websiteScraper.ts       # Scrape homepage for links
│   │   ├── patternTester.ts        # Test common URL patterns
│   │   ├── searchEngine.ts         # Google/DDG search
│   │   ├── jinaReader.ts           # Jina link extraction
│   │   └── llmAnalyzer.ts          # AI-powered discovery
│   ├── validators/
│   │   ├── linkValidator.ts        # HTTP checks, type validation
│   │   └── confidenceScorer.ts     # Calculate confidence scores
│   ├── categorizers/
│   │   └── linkCategorizer.ts      # Classify and normalize links
│   ├── writers/
│   │   └── frontmatterWriter.ts    # Update tool frontmatter
│   ├── state/
│   │   └── stateManager.ts         # Read/write state files
│   └── utils/
│       ├── logger.ts               # Structured logging
│       ├── retry.ts                # Retry logic
│       └── rateLimiter.ts          # Rate limiting
├── tests/
│   ├── discoverers/
│   ├── validators/
│   └── integration/
├── config/
│   └── default.yml                 # Global configuration
├── .env.example                    # API keys template
├── package.json
├── tsconfig.json
└── README.md

discovery:
  enabled_methods:
    - website_scraper
    - pattern_tester
    - jina_reader
    - llm_analyzer
  confidence_threshold: medium
  rate_limits:
    http_requests_per_second: 5
    concurrent_requests: 10
    jina_requests_per_day: 1000
    llm_requests_per_batch: 100
  retry:
    max_attempts: 3
    backoff_multiplier: 2
    initial_delay_ms: 1000
validation:
  check_http_status: true
  follow_redirects: true
  max_redirects: 3
  timeout_ms: 10000
  verify_ssl: true
output:
  frontmatter_fields:
    - linkedin_url
    - twitter_url
    - github_repo_url
    - github_org_url
    - youtube_channel_url
    - blog_rss_url
    - changelog_url
    - docs_url
  add_metadata: true
  dry_run: false
logging:
  level: info
  format: json
  file: logs/link-aggregator.log

# Discover links for a single tool
npm run discover -- --tool "tooling/AI-Toolkit/.../Trae AI.md"

# Batch process all tools
npm run discover -- --batch --directory "tooling/AI-Toolkit"

# Re-scan tools with low confidence
npm run discover -- --rescan --min-confidence low

# Dry run (preview without writing)
npm run discover -- --dry-run --tool "tooling/.../Cursor.md"

# Generate watch configurations
npm run discover -- --generate-watch-configs

# Human review mode
npm run review-queue

{
  "timestamp": "2025-11-15T10:30:00Z",
  "level": "info",
  "service": "link-aggregator",
  "tool": "trae-ai",
  "event": "links_discovered",
  "data": {
    "links_found": 12,
    "links_validated": 10,
    "confidence": "high",
    "discovery_time_ms": 8543,
    "methods_used": ["website_scraper", "jina_reader"]
  }
}

async function backfillLinks() {
  const tools = await loadAllTools();

  for (const tool of tools) {
    // 1. Preserve existing manual links
    const existingLinks = extractExistingLinks(tool.frontmatter);

    // 2. Discover new links
    const discoveredLinks = await discoverLinks(tool);

    // 3. Merge (manual links take precedence)
    const mergedLinks = {
      ...discoveredLinks,
      ...existingLinks,  // Overwrite with manual
      links_manually_verified: hasManualLinks(existingLinks)
    };

    // 4. Update frontmatter
    await updateFrontmatter(tool.path, mergedLinks);
  }
}

# Claude Desktop
cat ~/.config/claude/claude_desktop_config.json

# Cline (VS Code)
cat ~/.config/Code/User/globalStorage/saoudrizwan.claude-dev/settings/cline_mcp_settings.json

# Check for any MCP config files
find ~ -name "*mcp*.json" 2>/dev/null

// Step 1: Fetch homepage content with Jina Reader
const jinaUrl = `https://r.jina.ai/${productUrl}`;
const markdown = await fetch(jinaUrl).then(r => r.text());

// Step 2: Ask Claude to extract all links
const prompt = `Extract social media, GitHub, blog RSS, changelog, and documentation links from this content.
Product: ${productName}
Content: ${markdown}

Return as JSON with fields: linkedin_url, twitter_url, github_repo_url, youtube_channel_url, blog_rss_url, changelog_url, docs_url`;

const links = await claude.messages.create({
  model: "claude-haiku-20250514",
  messages: [{ role: "user", content: prompt }]
});

// Step 3: Validate URLs (simple HTTP check)
// Done!

const response = await perplexity.chat.completions.create({
  model: "llama-3.1-sonar-large-128k-online",
  messages: [{
    role: "user",
    content: `What are the official social media links, GitHub repositories, blog RSS feed, changelog, and documentation URLs for ${productName} (${productUrl})? Return as JSON.`
  }]
});

// Perplexity searches the web and returns structured answer with citations

npm install -g @spences10/mcp-omnisearch

# Add to Claude Desktop config
{
  "mcpServers": {
    "omnisearch": {
      "command": "npx",
      "args": ["-y", "@spences10/mcp-omnisearch"],
      "env": {
        "TAVILY_API_KEY": "your-key",
        "BRAVE_API_KEY": "your-key",
        "EXA_API_KEY": "your-key"
      }
    }
  }
}

// Search for company social links using Brave
await mcp.callTool("omnisearch", "brave_search", {
  query: '"Trae AI" site:linkedin.com OR site:twitter.com OR site:github.com'
});

// Or use Exa for semantic search
await mcp.callTool("omnisearch", "exa_search", {
  query: "Trae AI official social media profiles"
});

// mcp-server-link-aggregator/src/index.ts
import { Server } from "@modelcontextprotocol/sdk/server/index.js";

const server = new Server({
  name: "link-aggregator",
  version: "1.0.0"
}, {
  capabilities: {
    tools: {}
  }
});

server.setRequestHandler(ListToolsRequestSchema, async () => ({
  tools: [{
    name: "discover_product_links",
    description: "Discover social media, GitHub, blog, and operational links for a product",
    inputSchema: {
      type: "object",
      properties: {
        product_url: { type: "string", description: "Product homepage URL" },
        product_name: { type: "string", description: "Product name" }
      },
      required: ["product_url", "product_name"]
    }
  }]
}));

server.setRequestHandler(CallToolRequestSchema, async (request) => {
  if (request.params.name === "discover_product_links") {
    const { product_url, product_name } = request.params.arguments;

    // Use Jina + LLM or any discovery method
    const links = await discoverLinks(product_url, product_name);

    return {
      content: [{
        type: "text",
        text: JSON.stringify(links, null, 2)
      }]
    };
  }
});

// Claude Desktop, Cline, or any MCP client
const result = await mcp.callTool("link-aggregator", "discover_product_links", {
  product_url: "https://www.trae.ai/",
  product_name: "Trae"
});

// Query Wikidata for product
const wikidataId = await searchWikidata(productName);
const entity = await fetch(`https://www.wikidata.org/wiki/Special:EntityData/${wikidataId}.json`);

// Extract properties:
// P856: official website
// P2013: Facebook username
// P2002: Twitter username
// P1581: blog URL
// P1324: source code repository

Start Here
    ↓
Do you use Claude Desktop/Cline regularly?
    ↓                           ↓
   YES                         NO
    ↓                           ↓
MCP Omnisearch          Jina + Claude Script
(10 min setup)          (1 hour to build)
    ↓                           ↓
Test 50 tools            Test 50 tools
    ↓                           ↓
    └───── Good results? ───────┘
                ↓
               YES → Scale to 1,400 tools
                ↓
               NO → Try Firecrawl MCP or build custom solution

url: https://www.cursor.com/
site_name: Cursor

url: https://www.cursor.com/
site_name: Cursor
linkedin_url: https://www.linkedin.com/company/cursor-ai/
twitter_url: https://x.com/cursor_ai
github_repo_url: https://github.com/getcursor/cursor
youtube_channel_url: https://www.youtube.com/@cursor-ai
discord_url: https://discord.gg/cursor
docs_url: https://docs.cursor.com
blog_url: https://cursor.com/blog
changelog_url: https://changelog.cursor.com
links_last_updated: 2025-11-15
links_confidence: high
links_auto_discovered: true