paginating requests to an API, using HackerNews API - reactjs

Question

*** The question is quite simple, I just wrote a lot to be specific. ***
I've been looking online for a few hours, and I can't seem to find answer. Most pagination is about after you have received the data from the API call, or for backend node.js built with it's own server.
My issue, I have an API request that returns an array of 500 ID's. Then a second multi API call, looping through each ID making a promise API call. I use the Promise.all method.
It takes 2-3 minutes to complete this request.
Currently, I made a quick filter to get the first ten results, so it'll display and I can render the data to work on other things like the render component and styling.
My question, I'd like to be able to paginate the data while API calls are still being made.
Basically, Promise.all send an array of 10 id's (ten API calls), get continually. But after the first set of ten, I'd like to start receiving the data to render.
Right now, I can only get ten with my filter method. Or wait 2-3 min for all 500 to render.
Here is my request.js file, (it's part of my App component, I just separated it for clarity).
import axios from 'axios';
import BottleNeck from 'bottleneck'
const limiter = new BottleNeck({
maxConcurrent: 1,
minTime: 333
})
export const Request = (setResults, searchBarType, setLoading) => {
const searchBar = type => {
const obj = {
'new': 'newstories',
'past': '',
'comments': 'user',
'ask': 'askstories',
'show': 'showstories',
'jobs': 'jobstories',
'top': 'topstories',
'best': 'beststories',
'user': 'user'
}
return obj[type] ? obj[type] : obj['new'];
}
let type = searchBar(searchBarType)
const getData = () => {
const options = type
const API_URL = `https://hacker-news.firebaseio.com/v0/${options}.json?print=pretty`;
return new Promise((resolve, reject) => {
return resolve(axios.get(API_URL))
})
}
const getIdFromData = (dataId) => {
const API_URL = `https://hacker-news.firebaseio.com/v0/item/${dataId}.json?print=pretty`;
return new Promise((resolve, reject) => {
return resolve(axios.get(API_URL))
})
}
const runAsyncFunctions = async () => {
setLoading(true)
const {data} = await getData()
let firstTen = data.filter((d,i) => i < 10);
Promise.all(
firstTen.map(async (d) => {
const {data} = await limiter.schedule(() => getIdFromData(d))
console.log(data)
return data;
})
)
.then((newresults) => setResults((results) => [...results, ...newresults]))
setLoading(false)
// make conditional: check if searchBar type has changed, then clear array of results first
}
runAsyncFunctions()
}
and helps, here's my App.js file
import React, { useState, useEffect } from 'react';
import './App.css';
import { SearchBar } from './search-bar';
import { Results } from './results';
import { Request } from '../helper/request'
import { Pagination } from './pagination';
function App() {
const [results, setResults] = useState([]);
const [searchBarType, setsearchBarType] = useState('news');
const [loading, setLoading] = useState(false);
const [currentPage, setCurrentPage] = useState(1);
const [resultsPerPage] = useState(3);
// Select search bar button type
const handleClick = (e) => {
const serachBarButtonId = e.target.id;
console.log(serachBarButtonId)
setsearchBarType(serachBarButtonId)
}
// API calls
useEffect(() => {
Request(setResults, searchBarType, setLoading)
}, [searchBarType])
// Get current results
const indexOfLastResult = currentPage * resultsPerPage;
const indexOfFirstResult = indexOfLastResult - resultsPerPage;
const currentResults = results.slice(indexOfFirstResult, indexOfLastResult);
// Change page
const paginate = (pageNumber) => setCurrentPage(pageNumber);
return (
<div className="App">
<SearchBar handleClick={handleClick} />
<Results results={currentResults} loading={loading} />
<Pagination resultsPerPage={resultsPerPage} totalResults={results.length} paginate={paginate} />
</div>
);
}
export default App;
I hope it's generic looking enough to follow guide lines. Please ask me anything to help clarify. I've spent 8-10 hours searching and attempting to solve this...

Answer 1

You can continue with your filter, but you have to do some changes, for totalResults props of the component Pagination you have to set 500 rows so the user can select the page he wants because if you set 10 rows, the pages a user can select are 1,2,3,4, but we don't need that we need to put all pages 1 to 34 pages because we have 500 ids. The second point, we need to fetch data from the server by page with a page size equal to 3 we need to pass to Request startIndex and lastIndex to Request.
Request.js
import axios from 'axios';
import BottleNeck from 'bottleneck'
const limiter = new BottleNeck({
maxConcurrent: 1,
minTime: 333
})
export const Request = (setResults, searchBarType, setLoading, startIndex, lastIndex) => {
const searchBar = type => {
const obj = {
'new': 'newstories',
'past': '',
'comments': 'user',
'ask': 'askstories',
'show': 'showstories',
'jobs': 'jobstories',
'top': 'topstories',
'best': 'beststories',
'user': 'user'
}
return obj[type] ? obj[type] : obj['new'];
}
let type = searchBar(searchBarType)
const getData = () => {
const options = type
const API_URL = `https://hacker-news.firebaseio.com/v0/${options}.json?print=pretty`;
return new Promise((resolve, reject) => {
return resolve(axios.get(API_URL))
})
}
const getIdFromData = (dataId) => {
const API_URL = `https://hacker-news.firebaseio.com/v0/item/${dataId}.json?print=pretty`;
return new Promise((resolve, reject) => {
return resolve(axios.get(API_URL))
})
}
const runAsyncFunctions = async () => {
setLoading(true)
const {data} = await getData()
let ids = data.slice(firstIndex, lastIndex+1) // we select our ids by index
Promise.all(
ids.map(async (d) => {
const {data} = await limiter.schedule(() => getIdFromData(d))
console.log(data)
return data;
})
)
.then((newresults) => setResults((results) => [...results, ...newresults]))
setLoading(false)
// make conditional: check if searchBar type has changed, then clear array of results first
}
runAsyncFunctions()
}
App.js
import React, { useState, useEffect } from 'react';
import './App.css';
import { SearchBar } from './search-bar';
import { Results } from './results';
import { Request } from '../helper/request'
import { Pagination } from './pagination';
function App() {
const [results, setResults] = useState([]);
const [searchBarType, setsearchBarType] = useState('news');
const [loading, setLoading] = useState(false);
const [currentPage, setCurrentPage] = useState(1);
const [resultsPerPage] = useState(3);
// Select search bar button type
const handleClick = (e) => {
const serachBarButtonId = e.target.id;
console.log(serachBarButtonId)
setsearchBarType(serachBarButtonId)
}
// API calls
useEffect(() => {
Request(setResults, searchBarType, setLoading, 0, 2) //we fetch the first 3 articles
}, [searchBarType])
// Change page
const paginate = (pageNumber) => {
// Get current results
const indexOfLastResult = currentPage * resultsPerPage;
const indexOfFirstPost = indexOfLastResult - resultsPerPage;
Request(setResults, searchBarType, setLoading, indexOfFirstPost , indexOfLastResult) //we fetch the 3 articles of selected page
setCurrentPage(pageNumber);
}
return (
<div className="App">
<SearchBar handleClick={handleClick} />
<Results results={results} loading={loading} />
<Pagination resultsPerPage={resultsPerPage} totalResults={500} paginate={paginate} />
</div>
);
}
export default App;