Mern Data flow problems in ReactJS and Axios - reactjs

Question

I am an aspiring react developer and I am having some issues with my app.
I am trying to scrape the New York times for an assignment I have, and I can get my data with a search to log server-side, but I cant seem to pass it back. I can get my data by pushing it to a MongoDB then querying it in a separate process from the front end, but I don't want to do that.
I want to pass the object back up the stack to the client side. Does anyone know how I might accomplish that?
here is some of my code.
my dir structure:
here is the client folder structure:
here is my Home.jsx file clientside in /pages:
import React, { Component } from 'react';
import { Container, Row, Column } from '../../components/BootstrapGrid';
import API from '../../utils/API'
import {Input, FormBtn} from '../../components/Form'
class Home extends Component {
state = {
formInput: "",
posts: [],
}
loadArticles = (res) => {
console.log('res')
}
handleInputChange = event => {
const { name, value } = event.target;
this.setState({
[name]: value
});
};
handleFormSubmit = event => {
event.preventDefault();
let query = this.state.formInput
// console.log(query)
API.scrapeArticles(query)
// console.log(this.state)
};
render() {
return (
<Container>
<Row>
<Column>
</Column>
<Column>
<Input
value={this.state.formInput}
onChange={this.handleInputChange}
name="formInput"
placeholder="Search Query (required)"
/>
<FormBtn onClick={this.handleFormSubmit}>Scrape NYT API</FormBtn>
</Column>
</Row>
</Container>
);
}
}
export default Home;
here is my code calling the clientside api in client/utils/api/:
import axios from "axios";
export default {
// getPosts: function () {
// return axios.get('/api/posts')
// },
// savePost: function (postData) {
// return axios.post("/api/posts", postData);
// },
scrapeArticles: function (query) {
// console.log(query)
let queryData = {
query: query
}
return axios.post('/api/scraper', queryData)
}
};
here is my code from the backend routes/index.js being hit by axios (i think? Im honestly not sure how but i think this is the flow):
const path = require("path");
const router = require("express").Router();
const postsController = require('../controllers/postsController')
router.route("/")
.get(postsController.getAll)
.post(postsController.create);
router.route('/api/scraper')
.post(postsController.scraper)
.get(postsController.scraper)
// If no API routes are hit, send the React app
router.use(function (req, res) {
res.sendFile(path.join(__dirname, "../client/build/index.html"));
});
module.exports = router;
here is my controller that is referenced in the file above:
const scraper = require('../scraper')
const db = require('../models');
module.exports = {
create: function (req, res) {
db.Post
.create(req.body)
.then(dbmodel => res.json(dbmodel))
.catch(err => res.status(422).json(err))
},
getAll: function (req, res) {
db.Post
.find(req.query)
.sort({date: -1})
.then(dbModel => res.json(dbModel))
.catch(err => res.status(422).json(err))
},
scraper: function (req, res) {
let queryData = req.body.query
scraper(queryData)
},
scraperGet: function (req, res) {
scraper()
console.log(res.body)
}
}
and lastly, here is the scraper file on the backend:
const request = require('request');
const mongoose = require('mongoose');
const db = require('./models');
const scraper = (queryData) => {
console.log(`#scraper ${queryData}`)
let articleData = []
request.get({
url: "https://api.nytimes.com/svc/search/v2/articlesearch.json",
qs: {
'api-key': "-----------------------------",
"q" : queryData
},
}, function (err, response, body) {
body = JSON.parse(body);
let articles = body.response.docs
articles.forEach(element => {
// console.log(element)
let title= element.headline.main
let url = element.web_url
let synopsis = element.abstract
let snippet = element.snippet
let source = element.source
let pubDate = element.pub_date
let article = {
title: title,
url: url,
synopsis: synopsis,
snippet: snippet,
source: source,
pubDate: pubDate,
}
// console.log(article)
articleData.push(article)
db.Post.create({title:article.title}).then(article => {
console.log(article)
}).catch(err => {
console.log(err)
})
});
return articleData
});
}
module.exports = scraper;
So i know right now it is pushing to mongo. This is only because I couldn't figure out how to pass that data back just stored in a variable.
I really don't want to have to push all my results to the db and then make a query for them. I want to have a save article function that you only save the ones you actually want.

Answer 1

You should send articleData to the client and then get it in the client side using .then() method of a promise.
Something like this:
scraper: function (req, res) {
let queryData = req.body.query
const articleData = scraper(queryData)
// return your json to the client
res.json(articleData)
},
Then you should receive this data in the client side, like this:
handleFormSubmit = event => {
event.preventDefault()
let query = this.state.formInput
API.scrapeArticles(query)
.then(resp => {
this.setState({ posts: resp })
})
}