Hi I want to parallelize a process on 40 cpus, where each job runs 1 process.
I am using C and MPI with the TORQUE (PBS) scheduler on my cluster.
Here is my script.
#include <stdio.h>
#include <stdlib.h>
#include <mpi.h>
#include <unistd.h>
const char FILE_NAME[] = "/home/foo/c/mpi/cov_test/test_command.txt";
const char WRK_DIR[] = "/home/foo/c/mpi/cov_test";
char comm[39][256];
int main(int argc, char **argv)
{
int rank;
int size;
int count =0;
FILE *in_file;
char line[256];
char *pos;
in_file = fopen(FILE_NAME, "r");
if (in_file == NULL) {
printf("Cannot open %s\n", FILE_NAME);
exit(8);
}
if(in_file)
{
while(fgets(line, sizeof(line), in_file))
{ /* Remove newline at end of file *
* Maybe not the best way to do it? *
* mpicc compiler throws an error */
if((pos=strchr(line, '\n')) != NULL)
*pos = '\0';
strcpy(comm[count], line);
count++;
}
}
chdir(WRK_DIR);
chdir(WRK_DIR);
MPI_Init(&argc,&argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
/* Saw this on another SO posting */
int start = (rank*count)/size;
int end = ((rank+1)*count)/size;
for(int i= start; i < end; i++) {
printf("%s\n", comm[rank]);
/*system(comm[rank]);*/
}
MPI_Finalize();
}
The output i only the first command of the file "test_command.txt"
Here is my Torque submission file
#!/bin/bash
#PBS -q condo
#PBS -l walltime=01:00:00
#PBS -l nodes=5:ppn=8
#PBS -j oe
#PBS -o /home/foo/c/mpi/cov_test/pilot_mpi_out
#had to export my perl libraries
export PERL5LIB=/home/foo/myperl/lib/perl5:/home/foo/myperl/share/perl5:$PERL5LIB
cd $HOME/c/mpi/cov_test
/opt/openmpi/bin/mpirun -machinefile $PBS_NODEFILE -np 40 ./pilot_mpi_test-2
I'm new to C so I might be (probably) doing something ghastly. Thanks for your time :)
It should be printf("%s\n", comm[i]); instead of printf("%s\n", comm[rank]);.
From what I understand you are getting a single line of output when you expect 40. You may want to verify that you are getting output from processes other than process 0. I would add a
printf("Processes %d\n", rank);
just after the MPI_Comm_size line. If you don't see a line from each process something will need changed in your environment. I did a little looking but my GoogleFu has been failing me this morning. The other option is to pass the strings back to process 0 with MPI.
Related
I'm trying to write to text file with MPI but the file is not created.
I need only to write at the master (rank = 0), but nothing works.
It only working when I running the program in console (and save corrupt element) and not in Mpich2 and
I attached the code.
Thanks for helping.
/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
* (C) 2001 by Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
/* This is an interactive version of cpi */
#include <mpi.h>
#include <stdio.h>
#include <stdlib.h>
int main(int argc,char *argv[])
{
int namelen, numprocs, rank;
char processor_name[MPI_MAX_PROCESSOR_NAME];
MPI_Init(&argc,&argv);
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
MPI_Comm_size(MPI_COMM_WORLD,&numprocs);
MPI_Get_processor_name(processor_name,&namelen);
MPI_Status status;
FILE* f = fopen("test.txt","wb+");
if (rank == 0) {
for (int i=0; i < 5; i++){
fprintf(f,"%d \n",i);
}
fclose(f);
}
else {
// do nothing
}
MPI_Finalize();
return 0;
}
In the sample code you posted, all processes open the file and only process 0 closes it. Could you try the following modification ?
if (rank == 0) {
FILE* f = fopen("test.txt","wb+");
if(f==NULL){printf("failed to open file: permission issue ?\n");exit(1);}
for (int i=0; i < 5; i++){
fprintf(f,"%d \n",i);
}
fclose(f);
}
Since your code seems to come from the Argonne National Laboratory, I suppose that it is ran on a cluster using a particular file system.
The following code is based on yours. It makes use of MPI_File_open() and MPI_File_write() on a single process, using MPI_COMM_SELF.
/* This is an interactive version of cpi */
#include <mpi.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
int main(int argc,char *argv[])
{
int namelen, numprocs, rank;
char processor_name[MPI_MAX_PROCESSOR_NAME];
MPI_Init(&argc,&argv);
MPI_Comm_rank(MPI_COMM_WORLD,&rank);
MPI_Comm_size(MPI_COMM_WORLD,&numprocs);
MPI_Get_processor_name(processor_name,&namelen);
MPI_Status status;
MPI_File fh;
if (rank == 0) {
MPI_File_open(MPI_COMM_SELF, "test.txt",MPI_MODE_CREATE | MPI_MODE_WRONLY,MPI_INFO_NULL,&fh);
//FILE* f = fopen("test.txt","wb+");
//if(f==NULL){
//printf("failed to open file\n");exit(1);
//}
for (int i=0; i < 5; i++){
char buf[42];
//fprintf(f,"%d \n",i);
snprintf(buf,42,"%d \n",i);
MPI_File_write(fh,buf,strlen(buf), MPI_CHAR,&status);
}
// fclose(f);
MPI_File_close(&fh);
}
else {
// do nothing
}
MPI_Finalize();
return 0;
}
Please make sure that you hae the permission to write in the considered folder. Make sure that all nodes can access this folder ! Try some folders like your folder in /tmp or /scratch... Your cluster may have some sort of documentation somewhere telling you where you can write files !
I have a large directory of music which is listed in a file called op. I have been able to build a command which will randomly pick a song from the op file using some creative math with the nanosecond output from the date command. It works fine from the command line:
sed -n $((10#$(date +%N)%$(wc -l /shared/southpark/music/op|cut -d ' ' -f 1)))p /shared/southpark/music/op
I want to include this command in a c program and read the line in with popen.
#include <stdio.h>
#include <string.h>
int main (int argc, char *argv[])
{
char command[201];
char buf[501];
FILE *fp;
strcpy(command, "sed -n $((10#$(date +%N)%$(wc -l /shared/southpark/music/op|cut -d ' ' -f 1)))p /shared/southpark/music/op");
if((fp = popen(command, "r")) == NULL)
{
fprintf(stderr, "music_player: popen failed\n");
return(1);
}
if(fgets(buf, sizeof(buf), fp) == NULL)
{
fprintf(stderr, "music_player: fgets failed\n");
return(1);
}
printf("%s\n", buf);
pclose(fp);
return(0);
}
But when I run it, I get the following error:
sh: 1: arithmetic expression: expecting EOF: "10#271445839%2278"
music_player: fgets failed
How can I do this? I'm not understanding the error message.
popen executes your command using
/bin/sh -c "command"
and your sh doesn't understand the 10# base-conversion prefix. You've been running the command in bash previously.
To fix, you have two options:
Discard the unnecessary 10# prefix (it is the default) for sh compatibility
Use bash:
popen("bash -c 'command'", ...)
After trying and failing with both of nneonneo's options, I had to resort to placing the command in a bash script file and then I popen'ed the script. It gives me the desired results.
#include <stdio.h>
#include <string.h>
int main (int argc, char *argv[])
{
char command[201];
char buf[501];
FILE *fp;
strcpy(command, "/dea/testing/popen/get_file");
if((fp = popen(command, "r")) == NULL)
{
fprintf(stderr, "music_player: popen failed\n");
return(1);
}
if(fgets(buf, sizeof(buf), fp) == NULL)
{
fprintf(stderr, "music_player: fgets failed\n");
return(1);
}
printf("%s", buf);
pclose(fp);
return(0);
}
I am about building a bar for DWM (ubuntu linux), showing wifi details such as the ssid.
Thats my code:
#include <stdio.h>
#include <stdlib.h>
int main( int argc, char *argv[] )
{
FILE *fp;
int status;
char path[1035];
/* Open the command for reading. */
fp = popen("iwconfig", "r");
if (fp == NULL) {
printf("Failed to run command\n" );
exit;
}
char s[500];
/* Read the output a line at a time - output it. */
while (fgets(path, sizeof(path)-1, fp) != NULL) {
sprintf(s,"%s%s",s, path);
}
//printf("%s",s);
/* close */
pclose(fp);
char delimiter[1] = "s";
char *ptr;
ptr = strtok(s, delimiter);
printf("SSID: %s\n", ptr);
return 0;
}
i am getting overflowerrors and dont know what to do.
I dont think, thats a good way to get the ssid either... :/
Suggestions?
I would rather use direct information from the kernel (such as netdevice(7)) rather than calling a sub-process.
Maybe this header can help: http://lxr.free-electrons.com/source/include/linux/wireless.h
Edit: if you still want to use popen, which don't you just add a | grep Essid: ?
$ /sbin/ifconfig 2>/dev/null | grep ESSID | cut -d: -f2
"pink-panter"
I have to do the simulation of the command tee in C for Linux. How does tee work internally? It looks like a T-shaped pipe, so should I use a pipe? Is there a special kind of pipe?
tee takes stdin and copies the data stream to stdout as well as a file given as an option, it can be used in many very different situations.
An implementation in C is quite simple, just make a program that copies all data from stdin to stdout, but also use the same output statements for stdout on a file that you opened based on the command line argument.
basically in pseudo code:
file f = open(argv[1])
while (! end of file stdin) {
buffer = read stdin
write stdout buffer
write f buffer
}
close(f)
Note that you don't really have to do anything with pipes, your shell will sort out the pipes, the program only has to copy data from one stream to two others.
I finished the program!
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
main(int argc, char *argv[]){
FILE *fp, *fp1;
char buffer;
if(argc != 4){
printf("\nError");
printf("\nSintaxis: tee [archivo1] [archivo2]\n");
exit(0);
}
if(strcmp(argv[1], "tee") == 0){
fp = fopen(argv[2], "r");
fp1 = fopen(argv[3], "w");
printf("\Content in %s:\n", argv[2]);
while(!feof(fp)){
buffer = fgetc(fp);
fputc(buffer, fp1);
printf("%c", buffer);
}
printf("\n\n%s received %s\n", argv[3], argv[2]);
fclose(fp);
fclose(fp1);
}
else
printf("\nThe first argument have to be tee\n");
}
Here is some code I wrote about 20 years ago to implement TEE in Windows. I have been using this with various batch files since then. Note the flush command at the end of each line.
#include <stdio.h>
#include <share.h>
int main (int argc, char * argv[])
{
if (argc < 2 )
{
printf ("Error: No output file name given, example: theCmd 2>&1 |ltee outputFileName \n");
return 1;
}
FILE *Out = _fsopen(argv[argc-1], "a", _SH_DENYWR);
if (NULL == Out)
{
char buf[300];
sprintf_s(buf, 300, "Error openning %s", argv[argc-1]);
perror(buf);
return 1;
}
int ch;
while ( EOF != (ch=getchar()))
{
putchar(ch);
putc(ch, Out);
if ( '\n' == ch )
fflush(Out);
}
_flushall();
fclose(Out);
return 0;
}
I want to run the following command from a C program to read the system's CPU and memory use:
ps aux|awk 'NR > 0 { cpu +=$3; ram+=$4 }; END {print cpu,ram}'
I am trying to pass it to the execl command and after that read its output:
execl("/bin/ps", "/bin/ps", "aux|awk", "'NR > 0 { cpu +=$3; ram+=$4 }; END {print cpu,ram}'",(char *) 0);
But in the terminal I am getting the following error:
ERROR: Unsupported option (BSD syntax)
I would like to know how to properly pass awk as argument to execl?
You can't do this here this way.
The problem is that you want to execute several commands. execl is for executing a single command. The statement you have is using shell syntax (notably the | )
You will have better luck combining it all up in a single string and using the system(3) call.
Instead of running awk and parsing awk's output, you can do the filtering and summation in C, which often can quickly become much more convenient. (It's about the same for the exact command you have here.)
#include <errno.h>
#include <stdio.h>
void ps_cpumem(FILE* f, double* cpu_total, double* mem_total) {
for (;;) {
char buf[2000];
if (!fgets(buf, sizeof buf, f)) {
return;
}
double cpu, mem;
if (sscanf(buf, "%*s %*s %lf %lf", &cpu, &mem) == 2) {
*cpu_total += cpu;
*mem_total += mem;
}
}
}
int main() {
errno = 0;
FILE* ps = popen("ps aux", "r");
if (!ps) {
if (errno == 0) puts("popen: memory allocation failed");
else perror("popen");
return 1;
}
double cpu = 0, mem = 0;
ps_cpumem(ps, &cpu, &mem);
int rc = pclose(ps);
if (rc == -1) return 1;
printf("%%cpu: %5.1f\n" "%%mem: %5.1f\n", cpu, mem);
return 0;
}
However, you can run the full command through popen, as it executes a shell:
FILE* output = popen("ps aux | awk 'NR > 0 { cpu +=$3; ram+=$4 }; END {print cpu,ram}'", "r");
// read from output to do with as you like
As Will suggested, popen() is what you want for capturing output for subsequent use inside your program. However, if you truly are wanting to do an exec operation, you can use the shell to do your bidding via execl():
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>
int main(int argc, char **argv)
{
printf("%s: execl returned unexpectedly: %d", argv[0],
execl("/bin/sh", "/bin/sh", "-c",
"ps aux | awk 'NR >0 { cpu += $3; ram+=$4}; END {print cpu, ram}'",
NULL));
exit(1);
}