Embed Size (px)
Citation preview
Detecting Hidden File System Problems
Nicholas P. Cardo
National Energy Research Scientific Computing Center
Lawrence Berkeley National Laboratory
•File system problems can exist•Sometimes there are no errors•How to detect a non-error problem•How to do it quickly
The Dilemma
# grep -i lustre messages | grep -i error | wc –l5836 (6 days)
3
Words of Wisdom
“Strive for perfection in everything. Take the best that exists and make it better. If it does not exist, create it. Accept nothing nearly right or good enough.”
Sir Henry Royceco-founder of Rolls-Royce
4
•llapi_ping ….. ping lustre components•llapi_file_create … create a file w/stripes•llapi_file_get_stripe … read stripe info
Useful API Calls
5
Wanted: Dead or Alive
hc=/proc/fs/lustre/health_check
for node in $mds $ossdo
xx=`ssh $node cat $hc`if [ “$xx” != “healthy” ]then
echo “$node not healthy”fi
done
6
OST Ping
/proc/fs/lustre/osc
readdir
llapi_ping
ping failed
., ..,num_refs DT_DIR
rc=0
Y
Y
Y
Y
N
N
N
N
7
How About
dir = opendir("/proc/fs/lustre/osc");
while((d = readdir(dir)) != NULL) {
if (!strcmp(d->d_name,".")) continue; if (!strcmp(d->d_name,"..")) continue; if (!strcmp(d->d_name,"num_refs")) continue;
if ( d->d_type == DT_DIR ) { if ((llrc=llapi_ping("osc”,d->d_name)) != 0) fprintf(stderr,”problem %s\n”,osc); }}
8
Benchmarking
“benchmark: to study (as a competitor’s product or business practices) in order to improve the performance of one’s own company.”
www.webster.com
“Benchmarking is a process of comparing one’s business processes and performance metrics to industry bests and/or best practices from other industries. Dimensions typically measured are quality, time and cost, improvements from learning mean doing things better, faster, and cheaper.”
www.wikipedia.org
Nick’s Corollary BM1: benchmarks can make anything look good.
9
Create File
rc=0
get failed
stripe cnt
ost match
Y YY
N NN
llapi_file_create
llapi_file_get_stripe
ost mismatchcnt mismatch
10
Create Striped Filerc = llapi_file_create(fname,0,ostnum,1,0);
lum = malloc(LOV_EA_MAX(lum));
/* read back the stripe data */rc = llapi_file_get_stripe(fname,lum);if (!rc) { /* check the stripe count */ if (lum->lmm_stripe_count != 1) { fprintf(stderr,"%s: stripe count mismatch: %s\n", ProgName,fname); } else { /* check the ost number */ oi = lum->lmm_objects[lum->lmm_stripe_offset].l_ost_idx; if (oi != ostnum){ fprintf(stderr,"%s: ost mismatch: %s\n", ProgName,fname); } }}
11
Write Test
size
Start time
End time
MB/swrite
Y
N
12
time(&b_timval); /* start the clock */
for (cnt=0;cnt<fcnt;cnt++) write(ifd,pattern,sizeof(pattern));
time(&e_timval); /* stop the clock */
fsync(ifd); /* flush the cache */
/* * calculate write mega bytes per second */wmbs=(float)fcnt*sizeof(pattern)/(float)(e_timval-b_timval)/1048576;
13
Read Test
size
Start time
End time
MB/sread
Y
N
14
lseek(ifd,0,SEEK_SET); /* rewind */
time(&b_timval); /* start the clock */
for(cnt=0;cnt<fcnt;cnt++) read(ifd,buf,1024);
time(&e_timval); /* stop the clock */
/* * calculate read mega bytes per second */rmbs=(float)fcnt*sizeof(pattern)/(float)(e_timval-b_timval)/1048576.0;
15
Data Check
size
read
Y
N
seek 0
patternY N
Pattern mismatch
16
lseek(ifd,0,SEEK_SET);/* rewind */
/* * repeat the read, but this time check the results * to make sure that what we read matches what we wrote */for(cnt=0;cnt<fcnt;cnt++) { read(ifd,buf,sizeof(buf));
if(strncmp(buf,pattern,sizeof(pattern))) { fprintf(stderr,"%s: read/write mismatch: %s\n", ProgName,fname); rc = 1; break; }}
17
Get OSTs
dp = opendir("/proc/fs/lustre/lov"); while((de=readdir(dp)) != NULL) { /* we only want the requested file system */ if (strncmp(de->d_name,fsp,strlen(fsp))) continue;
sprintf(rc.fsname,"%s\0",fsname);
/* get the number of osts */ sprintf(pname,"/proc/fs/lustre/lov/%s/numobd\0",de->d_name);
ffd = fopen(pname,"r"); fscanf(ffd,"%d",&rc.numobd); fclose(ffd);
break;}
18
In a Nutshell
MPI_Init Create
Write
Verify
Read
Get OSTs
Scale OK
19
MPI_Init (&argc, &argv); MPI_Comm_rank (MPI_COMM_WORLD, &rank);MPI_Comm_size (MPI_COMM_WORLD, &size);/* Parse the command line options */while ((optchr=getopt(argc,argv,"f:s:t:")) != EOF) { switch (optchr) { case 'f': fsname = optarg; break; case 's': fsize = atoi(optarg); break; case 't': tstdir = optarg; break; default : MPI_Abort(MPI_COMM_WORLD,1); }}fs = getfsinfo(fsname);if (size != fs->numobd) { fprintf(stderr,"%s: tasks(%d) != osts(%d)\n", ProgName,size,fs->numobd); MPI_Abort(MPI_COMM_WORLD,1);}/* construct the name of the test file */sprintf(fname,"%s/testfile.%s.%d\0",tstdir,fs->fsname,rank);
rc = StripeFile(fname,rank); /*create the striped file */
if(!rc) rc = RDWR(fname,rank,fsize,fsname);/* read/write test */
if (!rc) unlink (fname); /* delete the test file */MPI_Finalize();
Case #1
0 5 10 15 20 25 30 35 40 45 50130.0
135.0
140.0
145.0
150.0
155.0
160.0
165.0
170.0
175.0
Scratch Write Scratch Read Scratch2 Write Scratch2 Read
21
Case #2
Scratch Write Scratch Read
22
Case #3
write read
23
Case #3b
write read after write after read
Thank You!
24
