|
|
| |
| import express from 'express'; |
| import cors from 'cors'; |
| import multer from 'multer'; |
| import { commit } from '@huggingface/hub'; |
| import path from 'path'; |
| import { fileURLToPath } from 'url'; |
|
|
| const __filename = fileURLToPath(import.meta.url); |
| const __dirname = path.dirname(__filename); |
|
|
| const app = express(); |
| const port = process.env.PORT || 3001; |
|
|
| app.use(cors()); |
| |
| app.use(express.json({ limit: '500mb' })); |
| app.use(express.urlencoded({ limit: '500mb', extended: true })); |
|
|
| |
| const storage = multer.memoryStorage(); |
| const upload = multer({ |
| storage: storage, |
| limits: { |
| fileSize: 500 * 1024 * 1024, |
| fieldSize: 500 * 1024 * 1024, |
| } |
| }); |
|
|
| const SERVER_CONFIG = { |
| TOKEN: process.env.HF_TOKEN || '', |
| REPO: process.env.HF_REPO || 'TwanAPI/DataTwan', |
| TYPE: 'dataset' |
| }; |
|
|
| app.post('/api/upload', upload.array('files'), async (req, res) => { |
| try { |
| const files = req.files; |
| let paths = req.body.paths; |
|
|
| |
| if (!Array.isArray(paths)) { |
| paths = [paths].filter(Boolean); |
| } |
|
|
| if (!files || files.length === 0) { |
| return res.status(400).json({ error: 'No files provided' }); |
| } |
|
|
| if (files.length !== paths.length) { |
| return res.status(400).json({ error: `Mismatch: ${files.length} files vs ${paths.length} paths` }); |
| } |
|
|
| if (!SERVER_CONFIG.TOKEN) { |
| return res.status(500).json({ error: 'Server misconfiguration: HF_TOKEN secret is missing.' }); |
| } |
|
|
| console.log(`[SERVER] Processing batch of ${files.length} files...`); |
|
|
| const operations = files.map((file, index) => ({ |
| operation: 'addOrUpdate', |
| path: paths[index], |
| content: new Blob([file.buffer]) |
| })); |
|
|
| |
| const response = await commit({ |
| credentials: { |
| accessToken: SERVER_CONFIG.TOKEN, |
| }, |
| repo: { |
| type: SERVER_CONFIG.TYPE, |
| name: SERVER_CONFIG.REPO |
| }, |
| operations: operations, |
| title: `Batch upload of ${files.length} files` |
| }); |
|
|
| const commitHash = response.oid; |
| const urlPrefix = "https://huggingface.co/datasets"; |
| const urls = paths.map(p => `${urlPrefix}/${SERVER_CONFIG.REPO}/blob/${commitHash}/${p}`); |
|
|
| console.log(`[SERVER] Batch Committed: ${commitHash}`); |
|
|
| res.json({ |
| success: true, |
| count: files.length, |
| urls: urls |
| }); |
|
|
| } catch (error) { |
| console.error('[SERVER] Error:', error); |
| res.status(500).json({ |
| success: false, |
| error: error.message || 'Internal Server Error' |
| }); |
| } |
| }); |
|
|
| app.use(express.static(path.join(__dirname, 'dist'))); |
|
|
| app.get('*', (req, res) => { |
| res.sendFile(path.join(__dirname, 'dist', 'index.html')); |
| }); |
|
|
| app.listen(port, () => { |
| console.log(`✅ Server running on port ${port} | Repo: ${SERVER_CONFIG.REPO}`); |
| }); |
|
|
