初始化 GitHub Trending 项目收藏器
功能：每日抓取 GitHub Top 10 趋势项目，展示中文描述、功能摘要与安装命令， 支持用户收藏，未收藏项目 7 天后自动删除。 技术栈：Node.js + Express + SQLite / React + Vite + Tailwind CSS Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
MikiVL
@@ -0,0 +1,15 @@
|
||||
const db = require('./db');
|
||||
|
||||
function cleanup() {
|
||||
const result = db.prepare(`
|
||||
DELETE FROM repositories
|
||||
WHERE is_favorited = 0
|
||||
AND date(first_fetched_at) <= date('now', '-7 days')
|
||||
`).run();
|
||||
|
||||
if (result.changes > 0) {
|
||||
console.log(`[cleanup] Deleted ${result.changes} unfavorited repo(s) older than 7 days`);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { cleanup };
|
||||
@@ -0,0 +1,37 @@
|
||||
const Database = require('better-sqlite3');
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
|
||||
const dataDir = path.join(__dirname, '../../data');
|
||||
if (!fs.existsSync(dataDir)) fs.mkdirSync(dataDir, { recursive: true });
|
||||
|
||||
const db = new Database(path.join(dataDir, 'github_trending.db'));
|
||||
|
||||
db.pragma('journal_mode = WAL');
|
||||
db.pragma('foreign_keys = ON');
|
||||
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS repositories (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
full_name TEXT NOT NULL UNIQUE,
|
||||
name TEXT NOT NULL,
|
||||
owner TEXT NOT NULL,
|
||||
description TEXT,
|
||||
language TEXT,
|
||||
stars INTEGER NOT NULL DEFAULT 0,
|
||||
html_url TEXT NOT NULL,
|
||||
avatar_url TEXT,
|
||||
features_summary TEXT,
|
||||
install_code TEXT,
|
||||
first_fetched_at TEXT NOT NULL,
|
||||
last_fetched_at TEXT NOT NULL,
|
||||
is_favorited INTEGER NOT NULL DEFAULT 0,
|
||||
favorited_at TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_repos_favorited ON repositories(is_favorited);
|
||||
CREATE INDEX IF NOT EXISTS idx_repos_fetched_at ON repositories(first_fetched_at);
|
||||
`);
|
||||
|
||||
module.exports = db;
|
||||
@@ -0,0 +1,14 @@
|
||||
const { scrape } = require('../scraper');
|
||||
const { cleanup } = require('../cleanup');
|
||||
|
||||
async function triggerScrape(req, res) {
|
||||
try {
|
||||
await scrape();
|
||||
cleanup();
|
||||
res.json({ success: true, message: 'Scrape completed' });
|
||||
} catch (err) {
|
||||
res.status(500).json({ error: err.message });
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { triggerScrape };
|
||||
@@ -0,0 +1,27 @@
|
||||
const db = require('../db');
|
||||
|
||||
function favorite(req, res) {
|
||||
const row = db.prepare('SELECT id FROM repositories WHERE id = ?').get(req.params.id);
|
||||
if (!row) return res.status(404).json({ error: 'Not found' });
|
||||
|
||||
db.prepare(`
|
||||
UPDATE repositories SET is_favorited = 1, favorited_at = datetime('now')
|
||||
WHERE id = ?
|
||||
`).run(req.params.id);
|
||||
|
||||
res.json({ success: true });
|
||||
}
|
||||
|
||||
function unfavorite(req, res) {
|
||||
const row = db.prepare('SELECT id FROM repositories WHERE id = ?').get(req.params.id);
|
||||
if (!row) return res.status(404).json({ error: 'Not found' });
|
||||
|
||||
db.prepare(`
|
||||
UPDATE repositories SET is_favorited = 0, favorited_at = NULL
|
||||
WHERE id = ?
|
||||
`).run(req.params.id);
|
||||
|
||||
res.json({ success: true });
|
||||
}
|
||||
|
||||
module.exports = { favorite, unfavorite };
|
||||
@@ -0,0 +1,59 @@
|
||||
const db = require('../db');
|
||||
|
||||
function computeDaysUntilDeletion(row) {
|
||||
if (row.is_favorited) return null;
|
||||
const first = new Date(row.first_fetched_at);
|
||||
const now = new Date();
|
||||
const daysSince = Math.floor((now - first) / (1000 * 60 * 60 * 24));
|
||||
return Math.max(0, 7 - daysSince);
|
||||
}
|
||||
|
||||
function formatRepo(row) {
|
||||
return { ...row, days_until_deletion: computeDaysUntilDeletion(row) };
|
||||
}
|
||||
|
||||
function getRepos(req, res) {
|
||||
const filter = req.query.filter || 'all';
|
||||
let rows;
|
||||
|
||||
if (filter === 'today') {
|
||||
rows = db.prepare(`
|
||||
SELECT * FROM repositories
|
||||
WHERE date(last_fetched_at) = date('now')
|
||||
ORDER BY stars DESC
|
||||
`).all();
|
||||
} else if (filter === 'favorites') {
|
||||
rows = db.prepare(`
|
||||
SELECT * FROM repositories
|
||||
WHERE is_favorited = 1
|
||||
ORDER BY favorited_at DESC
|
||||
`).all();
|
||||
} else {
|
||||
rows = db.prepare(`
|
||||
SELECT * FROM repositories
|
||||
ORDER BY stars DESC
|
||||
`).all();
|
||||
}
|
||||
|
||||
res.json(rows.map(formatRepo));
|
||||
}
|
||||
|
||||
function getRepo(req, res) {
|
||||
const row = db.prepare('SELECT * FROM repositories WHERE id = ?').get(req.params.id);
|
||||
if (!row) return res.status(404).json({ error: 'Not found' });
|
||||
res.json(formatRepo(row));
|
||||
}
|
||||
|
||||
function getStats(req, res) {
|
||||
const total = db.prepare('SELECT COUNT(*) as count FROM repositories').get().count;
|
||||
const favorited = db.prepare('SELECT COUNT(*) as count FROM repositories WHERE is_favorited = 1').get().count;
|
||||
const pendingDeletion = db.prepare(`
|
||||
SELECT COUNT(*) as count FROM repositories
|
||||
WHERE is_favorited = 0
|
||||
AND date(first_fetched_at) <= date('now', '-6 days')
|
||||
`).get().count;
|
||||
|
||||
res.json({ total, favorited, pendingDeletion });
|
||||
}
|
||||
|
||||
module.exports = { getRepos, getRepo, getStats };
|
||||
@@ -0,0 +1,23 @@
|
||||
const cron = require('node-cron');
|
||||
const { scrape } = require('./scraper');
|
||||
const { cleanup } = require('./cleanup');
|
||||
|
||||
async function runJob() {
|
||||
try {
|
||||
await scrape();
|
||||
cleanup();
|
||||
} catch (err) {
|
||||
console.error('[scheduler] Job failed:', err.message);
|
||||
}
|
||||
}
|
||||
|
||||
function initScheduler() {
|
||||
// Run daily at 2:00 AM
|
||||
cron.schedule('0 2 * * *', runJob);
|
||||
console.log('[scheduler] Daily scrape scheduled at 02:00');
|
||||
|
||||
// Run once immediately on startup
|
||||
runJob();
|
||||
}
|
||||
|
||||
module.exports = { initScheduler };
|
||||
@@ -0,0 +1,178 @@
|
||||
const axios = require('axios');
|
||||
const cheerio = require('cheerio');
|
||||
const db = require('./db');
|
||||
const { translate } = require('./translate');
|
||||
|
||||
const TRENDING_URL = 'https://github.com/trending?since=daily';
|
||||
const README_URL = (owner, repo) => `https://api.github.com/repos/${owner}/${repo}/readme`;
|
||||
|
||||
function buildHeaders() {
|
||||
const headers = { 'User-Agent': 'github-trending-app/1.0' };
|
||||
if (process.env.GITHUB_TOKEN) {
|
||||
headers['Authorization'] = `token ${process.env.GITHUB_TOKEN}`;
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
|
||||
async function fetchTrending() {
|
||||
const res = await axios.get(TRENDING_URL, {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.5',
|
||||
},
|
||||
timeout: 30000,
|
||||
});
|
||||
const $ = cheerio.load(res.data);
|
||||
const repos = [];
|
||||
|
||||
$('article.Box-row').slice(0, 10).each((_, el) => {
|
||||
const $el = $(el);
|
||||
const href = $el.find('h2 a').attr('href') || '';
|
||||
const parts = href.replace(/^\//, '').split('/').filter(Boolean);
|
||||
const [owner, name] = parts;
|
||||
if (!owner || !name) return;
|
||||
|
||||
const descEl = $el.find('p').first();
|
||||
const description = descEl.text().trim() || null;
|
||||
|
||||
const starsText = $el.find('a[href$="/stargazers"]').first().text().replace(/,/g, '').trim();
|
||||
const stars = parseInt(starsText, 10) || 0;
|
||||
|
||||
const language = $el.find('span[itemprop="programmingLanguage"]').text().trim() || null;
|
||||
|
||||
repos.push({ owner, name, full_name: `${owner}/${name}`, description, stars, language });
|
||||
});
|
||||
|
||||
return repos;
|
||||
}
|
||||
|
||||
async function fetchReadme(owner, repo) {
|
||||
try {
|
||||
const res = await axios.get(README_URL(owner, repo), {
|
||||
headers: buildHeaders(),
|
||||
timeout: 10000,
|
||||
});
|
||||
const content = Buffer.from(res.data.content, 'base64').toString('utf-8');
|
||||
return content;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// 候选标题关键词：英文 + 中文 + emoji前缀容忍
|
||||
const FEATURES_RE = /^#{1,3}[^#\n]*(?:features?|highlights?|overview|about|what is|why|introduction|capabilities|framework|skills?|development|功能|特性|特点|介绍|亮点)[^#\n]*/im;
|
||||
|
||||
function extractSection(readme, headingMatch) {
|
||||
const start = readme.indexOf(headingMatch) + headingMatch.length;
|
||||
const nextHeading = readme.slice(start).search(/^#{1,3}\s/m);
|
||||
const section = nextHeading === -1
|
||||
? readme.slice(start)
|
||||
: readme.slice(start, start + nextHeading);
|
||||
return section
|
||||
.replace(/```[\s\S]*?```/g, '')
|
||||
.replace(/!\[.*?\]\(.*?\)/g, '')
|
||||
.replace(/\[([^\]]+)\]\([^)]+\)/g, '$1')
|
||||
.replace(/[*_`#>\-|]/g, '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
.slice(0, 500) || null;
|
||||
}
|
||||
|
||||
function extractFeaturesSection(readme, description) {
|
||||
if (!readme) return description || null;
|
||||
|
||||
// 优先匹配功能相关标题
|
||||
const match = readme.match(FEATURES_RE);
|
||||
if (match) {
|
||||
const result = extractSection(readme, match[0]);
|
||||
if (result && result.length > 20) return result;
|
||||
}
|
||||
|
||||
// 兜底：取README正文第一段有意义的文字（跳过徽章行）
|
||||
const lines = readme.split('\n');
|
||||
const paragraphs = [];
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || trimmed.startsWith('#') || trimmed.startsWith('!') || trimmed.startsWith('[!') || trimmed.startsWith('<')) continue;
|
||||
if (trimmed.startsWith('[![') || trimmed.match(/^\[!\[/)) continue;
|
||||
if (trimmed.length > 40) paragraphs.push(trimmed);
|
||||
if (paragraphs.length >= 3) break;
|
||||
}
|
||||
const fallback = paragraphs.join(' ').replace(/\s+/g, ' ').trim().slice(0, 500);
|
||||
return fallback.length > 20 ? fallback : (description || null);
|
||||
}
|
||||
|
||||
function extractInstallCode(readme) {
|
||||
if (!readme) return null;
|
||||
const installMatch = readme.match(
|
||||
/#{1,3}\s*(?:install(?:ation)?|setup|getting[- ]started|quick[- ]start)\b[^\n]*/i
|
||||
);
|
||||
const searchFrom = installMatch
|
||||
? readme.indexOf(installMatch[0])
|
||||
: 0;
|
||||
const slice = readme.slice(searchFrom);
|
||||
const codeMatch = slice.match(/```[^\n]*\n([\s\S]*?)```/);
|
||||
return codeMatch ? codeMatch[1].trim() : null;
|
||||
}
|
||||
|
||||
async function fetchOwnerAvatar(owner) {
|
||||
try {
|
||||
const res = await axios.get(`https://api.github.com/users/${owner}`, {
|
||||
headers: buildHeaders(),
|
||||
timeout: 5000,
|
||||
});
|
||||
return res.data.avatar_url || null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
const upsert = db.prepare(`
|
||||
INSERT INTO repositories
|
||||
(full_name, name, owner, description, language, stars, html_url, avatar_url,
|
||||
features_summary, install_code, first_fetched_at, last_fetched_at)
|
||||
VALUES
|
||||
(@full_name, @name, @owner, @description, @language, @stars, @html_url, @avatar_url,
|
||||
@features_summary, @install_code, date('now'), date('now'))
|
||||
ON CONFLICT(full_name) DO UPDATE SET
|
||||
stars = excluded.stars,
|
||||
description = excluded.description,
|
||||
language = excluded.language,
|
||||
avatar_url = excluded.avatar_url,
|
||||
features_summary = excluded.features_summary,
|
||||
install_code = excluded.install_code,
|
||||
last_fetched_at = excluded.last_fetched_at
|
||||
`);
|
||||
|
||||
async function scrape() {
|
||||
console.log('[scraper] Starting scrape...');
|
||||
const repos = await fetchTrending();
|
||||
console.log(`[scraper] Found ${repos.length} trending repos`);
|
||||
|
||||
for (const repo of repos) {
|
||||
const readme = await fetchReadme(repo.owner, repo.name);
|
||||
const avatar_url = await fetchOwnerAvatar(repo.owner);
|
||||
const features_summary_raw = extractFeaturesSection(readme, repo.description);
|
||||
const install_code = extractInstallCode(readme);
|
||||
|
||||
const [description_zh, features_summary] = await Promise.all([
|
||||
translate(repo.description),
|
||||
translate(features_summary_raw),
|
||||
]);
|
||||
|
||||
upsert.run({
|
||||
...repo,
|
||||
description: description_zh,
|
||||
html_url: `https://github.com/${repo.full_name}`,
|
||||
avatar_url,
|
||||
features_summary,
|
||||
install_code,
|
||||
});
|
||||
console.log(`[scraper] Saved ${repo.full_name}`);
|
||||
}
|
||||
|
||||
console.log('[scraper] Done.');
|
||||
}
|
||||
|
||||
module.exports = { scrape };
|
||||
@@ -0,0 +1,33 @@
|
||||
require('dotenv').config({ path: require('path').join(__dirname, '../../.env') });
|
||||
|
||||
const express = require('express');
|
||||
const cors = require('cors');
|
||||
const path = require('path');
|
||||
const { getRepos, getRepo, getStats } = require('./routes/repos');
|
||||
const { favorite, unfavorite } = require('./routes/favorites');
|
||||
const { triggerScrape } = require('./routes/admin');
|
||||
const { initScheduler } = require('./scheduler');
|
||||
|
||||
const app = express();
|
||||
const PORT = process.env.PORT || 3001;
|
||||
|
||||
app.use(cors());
|
||||
app.use(express.json());
|
||||
|
||||
app.get('/api/repos', getRepos);
|
||||
app.get('/api/stats', getStats);
|
||||
app.get('/api/repos/:id', getRepo);
|
||||
app.post('/api/repos/:id/favorite', favorite);
|
||||
app.delete('/api/repos/:id/favorite', unfavorite);
|
||||
app.post('/api/scrape', triggerScrape);
|
||||
|
||||
const frontendDist = path.join(__dirname, '../../frontend/dist');
|
||||
app.use(express.static(frontendDist));
|
||||
app.get('*', (req, res) => {
|
||||
res.sendFile(path.join(frontendDist, 'index.html'));
|
||||
});
|
||||
|
||||
app.listen(PORT, () => {
|
||||
console.log(`[server] Running on http://localhost:${PORT}`);
|
||||
initScheduler();
|
||||
});
|
||||
@@ -0,0 +1,36 @@
|
||||
const axios = require('axios');
|
||||
|
||||
const CJK_RE = /[一-鿿㐀-䶿]/;
|
||||
const HTML_TAG_RE = /<[^>]{1,100}>/g;
|
||||
|
||||
function cleanText(text) {
|
||||
if (!text) return text;
|
||||
return text
|
||||
.replace(HTML_TAG_RE, ' ') // 去除 HTML 标签
|
||||
.replace(/&[a-z]+;/gi, ' ') // 去除 HTML 实体
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
async function translate(text) {
|
||||
const cleaned = cleanText(text);
|
||||
if (!cleaned || cleaned.length < 5) return cleaned || text;
|
||||
if (CJK_RE.test(cleaned)) return cleaned; // 已含中文，跳过
|
||||
|
||||
try {
|
||||
const res = await axios.get('https://api.mymemory.translated.net/get', {
|
||||
params: { q: cleaned.slice(0, 500), langpair: 'en|zh-CN' },
|
||||
headers: { 'User-Agent': 'github-trending-app/1.0' },
|
||||
timeout: 10000,
|
||||
});
|
||||
const translated = res.data?.responseData?.translatedText;
|
||||
if (!translated || translated === cleaned || translated.startsWith('PLEASE SELECT')) {
|
||||
return cleaned;
|
||||
}
|
||||
return translated.trim();
|
||||
} catch {
|
||||
return cleaned; // 失败静默降级，返回清理后原文
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { translate };
|
||||
Reference in New Issue
Block a user