lead-scraper/server.js

require('dotenv').config();
const express = require('express');
const path = require('path');
const { ApifyClient } = require('apify-client');
const store = require('./store');

const app = express();
const PORT = process.env.PORT || 8086;

app.set('view engine', 'ejs');
app.set('views', path.join(__dirname, 'views'));
app.use(express.static(path.join(__dirname, 'public')));
app.use(express.json());
app.use(express.urlencoded({ extended: true }));

// Increase timeout for long Apify runs
app.use((req, res, next) => {
  res.setTimeout(300000); // 5 min
  next();
});

// --- Pages ---
app.get('/', (req, res) => {
  const searches = store.getSearches();
  const searchList = Object.entries(searches)
    .map(([key, val]) => ({ key, ...val }))
    .sort((a, b) => (b.timestamp || 0) - (a.timestamp || 0));
  res.render('index', { searches: searchList, results: null, query: null, error: null });
});

app.get('/leads', (req, res) => {
  const leads = store.getLeads();
  const leadList = Object.entries(leads)
    .map(([key, val]) => ({ key, ...val }))
    .sort((a, b) => (b.savedAt || 0) - (a.savedAt || 0));
  res.render('leads', { leads: leadList });
});

app.get('/history', (req, res) => {
  const searches = store.getSearches();
  const searchList = Object.entries(searches)
    .map(([key, val]) => ({ key, ...val }))
    .sort((a, b) => (b.timestamp || 0) - (a.timestamp || 0));
  res.render('history', { searches: searchList });
});

// --- API ---
app.post('/api/search', async (req, res) => {
  const { keyword, city, state, maxResults } = req.body;
  if (!keyword || !city) return res.status(400).json({ error: 'Keyword and city are required' });
  
  const token = process.env.APIFY_API_TOKEN;
  if (!token) return res.status(500).json({ error: 'APIFY_API_TOKEN not configured. Add it to .env file.' });

  const searchQuery = `${keyword} in ${city}${state ? ', ' + state : ''}`;
  const max = parseInt(maxResults) || 20;

  try {
    const client = new ApifyClient({ token });
    const run = await client.actor('nwua9Gu5YrADL7ZDj').call({
      searchStringsArray: [searchQuery],
      maxCrawledPlacesPerSearch: max,
      language: 'en',
      deeperCityScrape: false,
    });

    const { items } = await client.dataset(run.defaultDatasetId).listItems();
    
    const results = items.map(item => ({
      placeId: item.placeId || item.cid || `place_${Date.now()}_${Math.random().toString(36).slice(2,8)}`,
      title: item.title || '',
      phone: item.phone || '',
      website: item.website || '',
      url: item.url || '',
      address: item.address || item.street || '',
      city: item.city || city,
      state: item.state || state || '',
      totalScore: item.totalScore || 0,
      reviewsCount: item.reviewsCount || 0,
      categoryName: item.categoryName || '',
      email: extractEmail(item),
      imageUrl: item.imageUrl || '',
      openingHours: item.openingHours || [],
      additionalInfo: item.additionalInfo || {},
      isBookmarked: store.isLead(item.placeId || item.cid || ''),
    }));

    const key = `search:${Date.now()}`;
    store.saveSearch(key, {
      query: searchQuery,
      keyword, city, state,
      maxResults: max,
      resultsCount: results.length,
      timestamp: Date.now(),
      results,
    });

    res.json({ success: true, key, results, query: searchQuery });
  } catch (err) {
    console.error('Apify error:', err);
    res.status(500).json({ error: `Scraping failed: ${err.message}` });
  }
});

app.get('/api/search/:key', (req, res) => {
  const search = store.getSearch(req.params.key);
  if (!search) return res.status(404).json({ error: 'Search not found' });
  // Refresh bookmark status
  if (search.results) {
    search.results = search.results.map(r => ({
      ...r,
      isBookmarked: store.isLead(r.placeId),
    }));
  }
  res.json(search);
});

app.delete('/api/search/:key', (req, res) => {
  store.deleteSearch(req.params.key);
  res.json({ success: true });
});

app.post('/api/leads', (req, res) => {
  const { place } = req.body;
  if (!place || !place.placeId) return res.status(400).json({ error: 'Invalid place data' });
  const key = `lead:${Date.now()}`;
  store.saveLead(key, { ...place, savedAt: Date.now() });
  res.json({ success: true, key });
});

app.delete('/api/leads/:placeId', (req, res) => {
  const leads = store.getLeads();
  const entry = Object.entries(leads).find(([, v]) => v.placeId === req.params.placeId);
  if (entry) store.deleteLead(entry[0]);
  res.json({ success: true });
});

app.get('/api/leads/check/:placeId', (req, res) => {
  res.json({ isLead: store.isLead(req.params.placeId) });
});

function extractEmail(item) {
  // Try to find email in various Apify output fields
  if (item.email) return item.email;
  if (item.emails && item.emails.length) return item.emails[0];
  const text = JSON.stringify(item.additionalInfo || {});
  const match = text.match(/[\w.-]+@[\w.-]+\.\w+/);
  return match ? match[0] : '';
}

app.listen(PORT, () => console.log(`Lead Scraper running on http://localhost:${PORT}`));