Is there a recommended/efficient way to convert a tf.data.Dataset to a Tensor when the underlying 'data examples' in the Dataset are flat arrays?
I am using tf.data.csv to read and parse a CSV but then want to use the Tensorflow.js Core API to process the data as tf.Tensors.
tf.data.Dataset.iterator() returns a promise of an iterator.
const it = await flattenedDataset.iterator()
const t = []
// read only the data for the first 5 rows
// all the data need not to be read once
// since it will consume a lot of memory
for (let i = 0; i < 5; i++) {
let e = await it.next()
t.push(...e.value)
}
tf.concat(await t, 0)
Using for await of
const asyncIterable = {
[Symbol.asyncIterator]() {
return {
i: 0,
async next() {
if (this.i < 5) {
this.i++
const e = await it.next()
return Promise.resolve({ value: e.value, done: false });
}
return Promise.resolve({ done: true });
}
};
}
};
const t = []
for await (let e of asyncIterable) {
if(e) {
t.push(e)
}
}
const csvUrl =
'https://storage.googleapis.com/tfjs-examples/multivariate-linear-regression/data/boston-housing-train.csv';
(async function run() {
// We want to predict the column "medv", which represents a median value of
// a home (in $1000s), so we mark it as a label.
const csvDataset = tf.data.csv(
csvUrl, {
columnConfigs: {
medv: {
isLabel: true
}
}
});
// Number of features is the number of column names minus one for the label
// column.
const numOfFeatures = (await csvDataset.columnNames()).length - 1;
// Prepare the Dataset for training.
const flattenedDataset =
csvDataset
.map(([rawFeatures, rawLabel]) =>
// Convert rows from object form (keyed by column name) to array form.
[...Object.values(rawFeatures), ...Object.values(rawLabel)])
.batch(1)
const it = await flattenedDataset.iterator()
const asyncIterable = {
[Symbol.asyncIterator]() {
return {
i: 0,
async next() {
if (this.i < 5) {
this.i++
const e = await it.next()
return Promise.resolve({ value: e.value, done: false });
}
return Promise.resolve({ done: true });
}
};
}
};
const t = []
for await (let e of asyncIterable) {
if(e) {
t.push(e)
}
}
console.log(tf.concat(t, 0).shape)
})()
<html>
<head>
<!-- Load TensorFlow.js -->
<script src="https://cdn.jsdelivr.net/npm/#tensorflow/tfjs#0.14.1"> </script>
</head>
<body>
</body>
</html>
Beware that this workflow is not typically recommended, because materializing
all the data in the main JavaScript memory may not work for large CSV datasets.
You can use the toArray() method of tf.data.Dataset objects. For example:
const csvUrl =
'https://storage.googleapis.com/tfjs-examples/multivariate-linear-regression/data/boston-housing-train.csv';
const csvDataset = tf.data.csv(
csvUrl, {
columnConfigs: {
medv: {
isLabel: true
}
}
}).batch(4);
const tensors = await csvDataset.toArray();
console.log(tensors.length);
console.log(tensors[0][0]);
Related
I'm trying to generate an array with values as follows:
{ name: 'John', age: 35, employer: 'ABC', paycheck: 5,000, last_paycheck: 4,900, change: 100 } // new array
with the initial values in the array as follow:
{ name: 'John', age: 35, employer: 'ABC' } //inital array
the function convertData() is handling all the array conversion.
async function convertData(data){
if(data.length === 0) return data;
// generates new array
const convertedDataArray = await data.map( async (row) =>{
let name = row.name
let paycheck = 0;
let last_paycheck = 0;
let change = 0;
const response = await axios.get('/getData', {params: {
name,
}});
let apiData = response.data.data;
if(apiData.length > 0){
let newData = apiData[0];
let oldData = apiData[1];
change = newData.payCheck - oldData.payCheck;
paycheck = newData.payCheck;
last_paycheck = oldData.payCheck;
}
console.log(apiData); // prints records up to 100 elements
return {...row, paycheck, last_paycheck, change };
});
console.log(convertedDataArray);// prints [Promise]
return Promise.all(convertedDataArray).then(() =>{
console.log(convertedDataArray); // prints [Promise]
return convertedDataArray;
});
};
where convertData() is called:
const response = await axios.get('/getEmployees',{params: {
token: id,
}});
const dataRows = response.data; //inital array
const tableRows = await convertData(dataRows);
return Promise.all(tableRows).then(() =>{
console.log(tableRows); // prints [Promise]
dispatch(setTableRows(tableRows));
});
I'm not sure why i keep getting Promise return I am still learning how to use promise correctly. Any help would be great, thank you in advance!
You should get a array of promises and use Promises.all to get all the data first.
Then use map() function to construct your data structure.
Example below:
async function convertData(data) {
try {
if (data.length === 0) return data;
const arrayOfPromises = data.map(row =>
axios.get("/getData", {
params: {
name: row.name,
},
})
);
const arrayOfData = await Promise.all(arrayOfPromises);
const convertedDataArray = arrayOfData.map((response, i) => {
const apiData = response.data.data;
let paycheck = 0;
let last_paycheck = 0;
let change = 0;
if (apiData.length > 0) {
const newData = apiData[0];
const oldData = apiData[1];
change = newData.payCheck - oldData.payCheck;
paycheck = newData.payCheck;
last_paycheck = oldData.payCheck;
}
return { ...data[i], paycheck, last_paycheck, change };
});
return convertedDataArray;
} catch (err) {
throw new Error(err);
}
}
(async function run() {
try {
const response = await axios.get("/getEmployees", {
params: {
token: id,
},
});
const dataRows = response.data;
const tableRows = await convertData(dataRows);
dispatch(setTableRows(tableRows));
} catch (err) {
console.log(err);
}
})();
Despite looking and following numerous answers here at stackoverflow,I have still failed to refactor this code to abide by the ESLint no-loop-func.
I keep getting the following warning, despite my efforts to refactor the code:
Compiled with warnings.
Function declared in a loop contains unsafe references to variable(s) 'lastResult', 'biologyBooks', 'page' no-loop-func
Here's the code:
import React from 'react';
import { apiFullCall } from '../../apiHelper';
const MyComponent = props => {
const [state, setState] = React.useState({ total: 0, biologyBooksByAuthor: [] });
let isLoaded = React.useRef(true);
const token = sessionStorage.getItem('token');
const authorID = sessionStorage.getItem('author_id');
const getBooks = async() => { // fetch items
let page = 1;
let scienceBooks, biologyBooks;
// create empty arrays to store book objects for each loop
let scienceBooks = biologyBooks = [];
// create a lastResult object to help check if there is a next page
let lastResult = { next: null };
do { // the looping - this is what I have failed to refactor
try {
await apiFullCall( // Make API calls over paginated records
'',
token,
'get',
`books/?author_id=1&page=${page}`
).then(res => {
if (res) {
const { status, body } = res;
if (status === 200 || status === 201) {
lastResult = body; // assign lastResult to pick "next"
body &&
body.results &&
body.results.map(eachBook => { // we map() over the returned "results" array
// the author with queried "author_id" writes science books;
// so we add each book (an object) into the science category
scienceBooks.push(eachBook);
// We then filter the author's biology books (from other science books)
biologyBooks = scienceBooks.filter(
({ is_biology }) =>
typeof(is_biology) === "boolean" && is_biology === true
);
return null;
}
);
// increment the page with 1 on each loop
page++;
}
}
}).catch(error => console.error('Error while fetching data:', error));
} catch (err) { console.error(`Oops, something went wrong ${err}`); }
// keep running until there's no next page
} while (lastResult.next !== null);
// update the state
setState(prevState => ({
...prevState, total: scienceBooks.length, biologyBooksByAuthor: biologyBooks,
}));
};
React.useEffect(() => { // fetch science books by author (logged in)
if (isLoaded && authorID) {
getBooks();
};
return function cleanup() {...}; // clean up API call, on unmount
}, [isLoaded, authorID]);
return (
// render the JSX code
);
}
Please note that I actually declared the said variables lastResult, biologyBooks and page outside the "do-while".
Any help or clues will be greatly appreciated.
The function the warning is referring to is the .then callback, if you're using async/await stick to it, try removing the .then part by assigning the result to a variable instead and remove the unnecessary .map, you can concatenate previous results with spread operator or .concat.
import React from 'react';
import { apiFullCall } from '../../apiHelper';
const MyComponent = props => {
const [state, setState] = React.useState({
total: 0,
scienceBooksByAuthor: [],
});
const isLoaded = React.useRef(true);
const token = sessionStorage.getItem('token');
const authorID = sessionStorage.getItem('author_id');
const getBooks = async () => {
// fetch items
let page = 1;
let scienceBooks = [];
// create a lastResult object to help check if there is a next page
let lastResult = { next: null };
do {
// the looping - this is what I have failed to refactor
try {
const res = await apiFullCall(
// Make API calls over paginated records
'',
token,
'get',
`books/?author_id=1&page=${page}`,
);
if (res) {
const { status, body } = res;
if (status === 200 || status === 201) {
lastResult = body; // assign lastResult to pick "next"
// concatenate new results
scienceBooks = [
...scienceBooks,
...((lastResult && lastResult.results) || []),
];
// increment the page with 1 on each loop
page += 1;
}
}
} catch (err) {
console.error(`Oops, something went wrong ${err}`);
}
// keep running until there's no next page
} while (lastResult.next !== null);
const biologyBooks = scienceBooks.filter(
({ is_biology }) =>
typeof is_biology === 'boolean' && is_biology === true,
);
// update the state
setState(prevState => ({
...prevState,
total: scienceBooks.length,
scienceBooksByAuthor: scienceBooks,
}));
};
React.useEffect(() => {
// fetch science books by author (logged in)
if (isLoaded && authorID) {
getBooks();
}
return function cleanup() {...}; // clean up API call, on unmount
}, [isLoaded, authorID]);
return (
// render the JSX code
);
};
I'm currently trying to get a project working to test some things and I'm stuck at a point where I'm trying to update the state properly.
I have an endpoint accessed via axios.get("/docker/containers") which will return an array for all IDs of the containers which are currently running on my system this is done like so:
componentDidMount() {
this.interval = setInterval(() => this.updateContainers(), 3000);
};
componentWillUnmount() {
clearInterval(this.interval);
}
At this point my state looks like this:
state = {
containers: [{id: 'id1'}, {id: 'id2'}]
}
The user interface then just shows a list of IDs.
I can then click on an ID on my user interface and it will set a watcher:
state = {
containers: [{id: 'id1', watcher: true}, {id: 'id2'}]
}
The point of the watcher is so that on the next update cycle more detailed information about a particular container is retrieved.
state = {
containers: [{id: 'id1', watcher: true, name: 'container1'}, {id: 'id2'}]
}
Upon clicking the container in the user interface where a watcher is already set then the watcher is dropped and the more detailed information is then no longer retrieved
state = {
containers: [{id: 'id1', watcher: false}, {id: 'id2'}]
}
Where I'm getting stuck is on how to get the more detailed information. My updateContainers method has 3 steps:
Read the response from the API and destruct the state into separate variables, compare the state var with the response var and remove any containers that have gone down (no setState is done here).
Add any new containers from the response to the state that have since come up (again no setState).
...All good thus far...
Loop through the filtered array of containers from steps 1 and 2 and find any containers where a watcher is set. Where it is set perform an API call to retrieve the more detailed info. Finally set the state.
In step 3 I use a forEach on the filtered array and then do an axios.get("/docker/containers/id1") where a watcher has been set otherwise simply keep the container details I already have but that's where I get stuck, Typescript is also giving me the error:
TS2322: Type 'void' is not assignable to type 'IndividualContainer[]'.
currently I have:
updateContainers() {
axios.get('/docker/containers')
.then(response => {
const apiRequestedContainers: string[] = response.data.containers;
// array of only IDs
const stateContainers: IndividualContainer[] = [
...this.state.containers
];
// remove dead containers from state by copying still live containers
let filteredContainers: IndividualContainer[] = [
...this.filterOutContainers(stateContainers, apiRequestedContainers)
];
// add new containers
filteredContainers = this.addContainerToArray(
filteredContainers, apiRequestedContainers
);
return this.updateContainer(filteredContainers);
})
.then(finalArray => {
const newState: CState = {'containers': finalArray};
this.setState(newState);
});
};
updateContainer(containers: IndividualContainer[]) {
const returnArray: IndividualContainer[] = [];
containers.forEach(container => {
if (container.watcher) {
axios.get('/docker/containers/' + container.id)
.then(response => {
// read currently available array of containers into an array
const resp = response.data;
resp['id'] = container.id;
resp['watcher'] = true;
returnArray.push(resp);
});
} else {
returnArray.push(container);
}
return returnArray;
});
};
Any pointers to where my logic fails would be appreciated!
Edit:
Render Method:
render() {
const containers: any = [];
const curStateOfContainers: IndividualContainer[] = [...this.state.containers];
if (curStateOfContainers.length > 0) {
curStateOfContainers.map(container => {
const container_id = container.id.slice(0, 12);
containers.push(
<Container
key = {container_id}
container_id = {container.id}
name = {container.name}
clickHandler = {() => this.setWatcher(container.id)}
/>
);
});
}
return containers;
}
I'm not an expert in TypeScript so I had to change the response to JS and thought you'll re-write it in TS in case it's needed.
async updateContainers() {
const response = await axios.get('/docker/containers')
const apiRequestedContainers = response.data.containers; // array of only IDs
const stateContainers = [...this.state.containers];
// remove dead containers from state by copying still live containers
let filteredContainers = [...this.filterOutContainers(stateContainers, apiRequestedContainers)];
// add new containers
filteredContainers = this.addContainerToArray(filteredContainers, apiRequestedContainers);
const containers = await this.updateContainer(filteredContainers)
this.setState({ containers });
};
async updateContainer(containers) {
return containers.map(async (container) => {
if (container.watcher) {
const response = await axios.get('/docker/containers/' + container.id)
// read currently available array of containers into an array
return {
...response.data,
id: container.id,
watcher: true,
}
} else {
return container;
}
});
}
Here's what I've updated in updateContainer:
I'm now mapping the array instead of doing a forEach
I'm now waiting for the container details API to return a value before checking the second container. --> this was the main issue as your code doesn't wait for the API to finish ( await / async )
The problem is that you are returning nothing from updateContainer method which will return void implicitly:
// This function return void
updateContainer(containers: IndividualContainer[]) {
const returnArray: IndividualContainer[] = [];
containers.forEach(container => {
if (container.watcher) {
axios.get("/docker/containers/" + container.id).then(response => {
// read currently available array of containers into an array
const resp = response.data;
resp["id"] = container.id;
resp["watcher"] = true;
returnArray.push(resp);
});
} else {
returnArray.push(container);
}
// this is inside the forEach callback function not updateContainer function
return returnArray;
});
}
Then you assign void to containers which is supposed to be of type IndividualContainer[] so TypeScript gives you an error then you set that in the state:
updateContainers() {
axios
.get("/docker/containers")
.then(response => {
const apiRequestedContainers: string[] = response.data.containers; // array of only IDs
const stateContainers: IndividualContainer[] = [
...this.state.containers
];
// remove dead containers from state by copying still live containers
let filteredContainers: IndividualContainer[] = [
...this.filterOutContainers(stateContainers, apiRequestedContainers)
];
// add new containers
filteredContainers = this.addContainerToArray(
filteredContainers,
apiRequestedContainers
);
// this return void as well
return this.updateContainer(filteredContainers);
})
// finalArray is void
.then(finalArray => {
// you assign void to containers which should be of type IndividualContainer[]
const newState: CState = { containers: finalArray };
// containers will be set to undefined in you state
this.setState(newState);
});
}
You meant to do this:
// I added a return type here so that TypeScript would yell at me if I return void or wrong type
updateContainer(containers: IndividualContainer[]): IndividualContainer[] {
const returnArray: IndividualContainer[] = [];
containers.forEach(container => {
if (container.watcher) {
axios.get("/docker/containers/" + container.id).then(response => {
// read currently available array of containers into an array
const resp = response.data;
resp["id"] = container.id;
resp["watcher"] = true;
returnArray.push(resp);
});
} else {
returnArray.push(container);
}
// removed the return from here as it's useless
});
// you should return the array here
return returnArray;
}
First, I've commented on errors in your code:
updateContainers() {
axios.get('/docker/containers')
.then(response => {
...
return this.updateContainer(filteredContainers);
// returns `undefined`...
})
.then(finalArray => { ... });
// ...so `finalArray` is `undefined` - the reason for TS error
// Also `undefined` is not a `Promise` so this second `then()`
// doesn't make much sense
};
updateContainer(containers: IndividualContainer[]) {
const returnArray: IndividualContainer[] = [];
containers.forEach(container => {
if (container.watcher) {
axios.get('/docker/containers/' + container.id)
.then(response => {
...
returnArray.push(resp)
// because `axios.get()` is asynchronous
// this happens only some time after
// `.then(finalArray => { ... })` is finished
});
// at this moment code inside `.then()` has not been executed yet
// and `resp` has not yet been added to `returnArray`
} else {
returnArray.push(container)
// but this happens while `forEach()` is running
}
return returnArray;
// here you return from `forEach()` not from `updateContainer()`
// also `forEach()` always returns `undefined`
// so even `return containers.forEach(...)` won't work
});
// no return statement, that implicitly means `return undefined`
};
Now, why the #RocKhalil's answer, kind of, works:
async updateContainers() {
const response = await axios.get('/docker/containers')
// he favors a much clearer syntax of async/await
...
const containers = await this.updateContainer(filteredContainers)
this.setState({ containers });
};
async updateContainer(containers) {
return containers.map(async (container) => {
if (container.watcher) {
const response = await axios.get('/docker/containers/' + container.id)
// Because `axios.get()` was **awaited**,
// you can be sure that all code after this line
// executed when the request ended
// while this
// axios.get(...).then(() => console.log(2)); console.log(1)
// will lead to output 1 2, not 2 1
return {
...response.data,
id: container.id,
watcher: true,
}
} else {
return container;
}
});
// he does not forget to return the result of `map()`
// and `map()` in contrast with `forEach()` does have a result
// But...
}
But...
containers.map() returns an array. An array of Promises. Not a single Promise. And that means that
const containers = await this.updateContainer(filteredContainers)
waits for nothing. And updateContainer() function is not actually async.
To fix that you need to use Promise.all():
const containers = await Promise.all(this.updateContainer(filteredContainers))
I got a little problem with synchronous/asynchronous system in the function "Array.foreach".
I don't know how to force my code to wait its end.
I tried to use await/async system but my code did not wait the code in "async responseDB =>".
This is my class:
...
let responsesDTO = [];
await Array.prototype.forEach.call(await searchResponsesByQuestionAndUserId(questions[cpt].idquestion, idUser), async responseDB => {
if(responseDB !== undefined){
const responseDTO = {
response_id:0,
response_text:"",
response_type:""
}
const responseEntity = await searchResponseByResponseId(responseDB.response_id);
responseDTO.response_id = responseDB.response_id;
responseDTO.response_text= responseEntity.text;
responseDTO.response_type= responseDB.type;
responsesDTO.push(responseDTO);
}
});
questionResponse.responses=responsesDTO;
questionResponses[cpt]=questionResponse;
}
Could you help me please? Thanks in advance.
I had to mock your async functions. However, the relevant part is to use for..of instead of forEach
async function searchResponsesByQuestionAndUserId() {
let responsesDB = [];
for (let i = 0; i < 10; i++) {
responsesDB.push({
response_id: parseInt(1000 * Math.random(), 10),
type: 'whatever ' + i
});
}
return new Promise((res) => {
window.setTimeout(() => {
res(responsesDB);
}, 1500);
});
}
async function searchResponseByResponseId(response_id) {
return new Promise((res) => {
window.setTimeout(() => {
res({
text: 'text for response ' + response_id
});
}, 300);
});
}
async function getResponsesDTO() {
let responsesDTO = [],
responsesDB = await searchResponsesByQuestionAndUserId();
for (let responseDB of responsesDB) {
if (responseDB === undefined) {
continue;
}
let responseDTO = {
response_id: 0,
response_text: "",
response_type: ""
},
responseEntity = await searchResponseByResponseId(responseDB.response_id);
responseDTO.response_id = responseDB.response_id;
responseDTO.response_text = responseEntity.text;
responseDTO.response_type = responseDB.type;
responsesDTO.push(responseDTO);
console.log({responseDTO});
}
return responsesDTO;
}
getResponsesDTO().then(responsesDTO => {
console.log(responsesDTO);
});
I have a React form where the user can upload multiple files. These are stored in fileList
async function uploadFiles(id) {
try {
const meta = await storageUploadFile(fileList, id);
console.log(meta);
} catch (e) {
console.log(e);
}
}
This calls my helper function that uploads the files to Firebase
export const storageUploadFile = function(files, id) {
const user = firebase.auth().currentUser.uid;
return Promise.all(
files.map((file) => {
return storage.child(`designs/${user}/${id}/${file.name}`).put(file)
})
)
};
What I'd like is on calling uploadFiles, get the total filesize of all items, and then show the overall progress.
At the moment, my code is only returning the file status in an array on completion
[
{bytesTransferred: 485561, totalBytes: 485561, state: "success"},
{bytesTransferred: 656289, totalBytes: 656289, state: "success"}
]
This is the way i do it:
import Deferred from 'es6-deferred';
export const storageUploadFile = function(files, id) {
const user = firebase.auth().currentUser.uid;
// To track the remaining files
let itemsCount = files.length;
// To store our files refs
const thumbRef = [];
// Our main tasks
const tumbUploadTask = [];
// This will store our primses
const thumbCompleter = [];
for (let i = 0; i < files.length; i += 1) {
thumbRef[i] = storage.ref(`designs/${user}/${id}/${file.name}`);
tumbUploadTask[i] = thumbRef[i].put(files[i]);
thumbCompleter[i] = new Deferred();
tumbUploadTask[i].on('state_changed',
(snap) => {
// Here you can check the progress
console.log(i, (snap.bytesTransferred / snap.totalBytes) * 100);
},
(error) => {
thumbCompleter[i].reject(error);
}, () => {
const url = tumbUploadTask[i].snapshot.metadata.downloadURLs[0];
itemsCount -= 1;
console.log(`Items left: ${itemsCount}`)
thumbCompleter[i].resolve(url);
});
}
return Promise.all(thumbCompleter).then((urls) => {
// Here we can see our files urls
console.log(urls);
});
};
Hope it helps.