NVMe SSD disk on AWS Sagemaker Notebook Instances - amazon-sagemaker

Sagemake Studio added support for instances with attached high-performance SSDs. Among others, the M5d instance features an attached SSD. (See, for instance here AWS Blog: Amazon SageMaker now supports M5d, R5, P3dn, and G4dn instances for SageMaker Notebook Instances)
My question is, how do I make use of the SSD?
The block device is not mounted when I start a Notebook with a Kernel on an M5d instance.
Below is a listing of the mount command form inside the notebook. I can see it when executing lsblk. (output is also below). However, there is no way I can format or mount the disk as the environment is restricted by Linux capabilities.
!mount
overlay on / type overlay (rw,relatime,lowerdir=/var/lib/dataroot/data-root/200010.1001/overlay2/l/JGWGMF4ZZ6PHEWK6JM4VICT7E4:/var/lib/dataroot/data-root/200010.1001/overlay2/l/66JWDG6GVXACNLSFJJBEOVZUO4:/var/lib/dataroot/data-root/200010.1001/overlay2/l/3U4YS24OOOKWXXZ7GZ3N4YOJZU:/var/lib/dataroot/data-root/200010.1001/overlay2/l/XOM2F2QYQYJMEUDYRFWJF637G2:/var/lib/dataroot/data-root/200010.1001/overlay2/l/4SSLKXEKJRW2M3MDUY6GRWQZOR:/var/lib/dataroot/data-root/200010.1001/overlay2/l/MDSB4ZKIAUXBZ3U5S4YM4DDOLQ:/var/lib/dataroot/data-root/200010.1001/overlay2/l/NE42TLPPST4P26YJZXL7KQYD5V:/var/lib/dataroot/data-root/200010.1001/overlay2/l/ZQA5ST7XAT673MNTWDL4YWBHLL:/var/lib/dataroot/data-root/200010.1001/overlay2/l/UZYHAWYHUINESI25LGXT5V2RHY:/var/lib/dataroot/data-root/200010.1001/overlay2/l/QECAGRX3FEW5FAJXK5DACNEFKA:/var/lib/dataroot/data-root/200010.1001/overlay2/l/IBZBCWE3M7DNAUXEOAHQUCM5HG:/var/lib/dataroot/data-root/200010.1001/overlay2/l/VQMIS7KW6ZB5VOQNKQE3J2RF4J:/var/lib/dataroot/data-root/200010.1001/overlay2/l/UOWDHH53G2E442U6NNUMBITSRJ,upperdir=/var/lib/dataroot/data-root/200010.1001/overlay2/9b5b78d144189e2437ee3a1716d2215d6a101155c2b35f9e8c0c3b1004b627fd/diff,workdir=/var/lib/dataroot/data-root/200010.1001/overlay2/9b5b78d144189e2437ee3a1716d2215d6a101155c2b35f9e8c0c3b1004b627fd/work)
proc on /proc type proc (rw,nosuid,nodev,noexec,relatime)
tmpfs on /dev type tmpfs (rw,nosuid,size=65536k,mode=755,uid=200010,gid=1001)
devpts on /dev/pts type devpts (rw,nosuid,noexec,relatime,gid=10000004,mode=620,ptmxmode=666)
sysfs on /sys type sysfs (ro,nosuid,nodev,noexec,relatime)
tmpfs on /sys/fs/cgroup type tmpfs (rw,nosuid,nodev,noexec,relatime,mode=755,uid=200010,gid=1001)
cgroup on /sys/fs/cgroup/systemd type cgroup (rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/usr/lib/systemd/systemd-cgroups-agent,name=systemd)
cgroup on /sys/fs/cgroup/blkio type cgroup (rw,nosuid,nodev,noexec,relatime,blkio)
cgroup on /sys/fs/cgroup/net_cls,net_prio type cgroup (rw,nosuid,nodev,noexec,relatime,net_cls,net_prio)
cgroup on /sys/fs/cgroup/cpuset type cgroup (rw,nosuid,nodev,noexec,relatime,cpuset)
cgroup on /sys/fs/cgroup/memory type cgroup (rw,nosuid,nodev,noexec,relatime,memory)
cgroup on /sys/fs/cgroup/perf_event type cgroup (rw,nosuid,nodev,noexec,relatime,perf_event)
cgroup on /sys/fs/cgroup/cpu,cpuacct type cgroup (rw,nosuid,nodev,noexec,relatime,cpu,cpuacct)
cgroup on /sys/fs/cgroup/freezer type cgroup (rw,nosuid,nodev,noexec,relatime,freezer)
cgroup on /sys/fs/cgroup/hugetlb type cgroup (rw,nosuid,nodev,noexec,relatime,hugetlb)
cgroup on /sys/fs/cgroup/pids type cgroup (rw,nosuid,nodev,noexec,relatime,pids)
cgroup on /sys/fs/cgroup/devices type cgroup (rw,nosuid,nodev,noexec,relatime,devices)
mqueue on /dev/mqueue type mqueue (rw,nosuid,nodev,noexec,relatime)
shm on /dev/shm type tmpfs (rw,nosuid,nodev,noexec,relatime,size=65536k,uid=200010,gid=1001)
127.0.0.1:/200010 on /root type nfs4 (rw,relatime,vers=4.1,rsize=1048576,wsize=1048576,namlen=255,hard,noresvport,proto=tcp,port=20486,timeo=600,retrans=2,sec=sys,clientaddr=127.0.0.1,local_lock=none,addr=127.0.0.1)
/dev/nvme0n1p1 on /opt/.sagemakerinternal type xfs (rw,noatime,attr2,inode64,usrquota,prjquota)
/dev/nvme0n1p1 on /etc/resolv.conf type xfs (rw,noatime,attr2,inode64,usrquota,prjquota)
/dev/nvme0n1p1 on /etc/hostname type xfs (rw,noatime,attr2,inode64,usrquota,prjquota)
/dev/nvme0n1p1 on /etc/hosts type xfs (rw,noatime,attr2,inode64,usrquota,prjquota)
/dev/nvme0n1p1 on /var/log/studio type xfs (rw,noatime,attr2,inode64,usrquota,prjquota)
/dev/nvme0n1p1 on /var/log/apps type xfs (rw,noatime,attr2,inode64,usrquota,prjquota)
/dev/nvme0n1p1 on /opt/ml/metadata/resource-metadata.json type xfs (ro,noatime,attr2,inode64,usrquota,prjquota)
devtmpfs on /dev/null type devtmpfs (rw,nosuid,size=16042360k,nr_inodes=4010590,mode=755)
devtmpfs on /dev/random type devtmpfs (rw,nosuid,size=16042360k,nr_inodes=4010590,mode=755)
devtmpfs on /dev/full type devtmpfs (rw,nosuid,size=16042360k,nr_inodes=4010590,mode=755)
devtmpfs on /dev/tty type devtmpfs (rw,nosuid,size=16042360k,nr_inodes=4010590,mode=755)
devtmpfs on /dev/zero type devtmpfs (rw,nosuid,size=16042360k,nr_inodes=4010590,mode=755)
devtmpfs on /dev/urandom type devtmpfs (rw,nosuid,size=16042360k,nr_inodes=4010590,mode=755)
proc on /proc/bus type proc (ro,nosuid,nodev,noexec,relatime)
proc on /proc/fs type proc (ro,nosuid,nodev,noexec,relatime)
proc on /proc/irq type proc (ro,nosuid,nodev,noexec,relatime)
proc on /proc/sys type proc (ro,nosuid,nodev,noexec,relatime)
proc on /proc/sysrq-trigger type proc (ro,nosuid,nodev,noexec,relatime)
tmpfs on /proc/acpi type tmpfs (ro,relatime,uid=200010,gid=1001)
devtmpfs on /proc/kcore type devtmpfs (rw,nosuid,size=16042360k,nr_inodes=4010590,mode=755)
devtmpfs on /proc/keys type devtmpfs (rw,nosuid,size=16042360k,nr_inodes=4010590,mode=755)
devtmpfs on /proc/latency_stats type devtmpfs (rw,nosuid,size=16042360k,nr_inodes=4010590,mode=755)
devtmpfs on /proc/timer_list type devtmpfs (rw,nosuid,size=16042360k,nr_inodes=4010590,mode=755)
devtmpfs on /proc/sched_debug type devtmpfs (rw,nosuid,size=16042360k,nr_inodes=4010590,mode=755)
tmpfs on /sys/firmware type tmpfs (ro,relatime,uid=200010,gid=1001)
!lsblk
NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
nvme1n1 259:0 0 279.4G 0 disk
nvme0n1 259:1 0 124G 0 disk
├─nvme0n1p1 259:2 0 124G 0 part /opt/ml/metadata/resource-metadata.json
└─nvme0n1p128 259:3 0 1M 0 part

Related

Why is gdb aborting when I try to print a cosine?

Here's my interaction with it. I first start gdb, set a breakpoint, run the program gdb halts at the breakpoint. Then:
<code>
(gdb) b 89
Breakpoint 1 at 0x18cc: file parseGaia3DataToSqDeg.c, line 89.
(gdb) r
Starting program: /sixTB/astro/catalogs/gaia3/shSqDeg/fj
Star 0.0281655 -89.857 not found in 0 tries.
Breakpoint 1, main (argc=1, argv=0x7fffffffe5c8) at parseGaia3DataToSqDeg.c:89
89 exit(0); //TEST
(gdb) p cos(.333)
Abort
</code>
Gdb simply quits, and I'm back at my command line.
Data on gdb:
gdb --version
GNU gdb (Debian 10.1-1.7) 10.1.90.20210103-git
My machine:
total used free shared buff/cache available
Mem: 27Gi 3.1Gi 1.2Gi 123Mi 23Gi 23Gi
Swap: 976Mi 3.0Mi 973Mi
CPU family: 25
AMD Ryzen 5 5600G with Radeon Graphics
CPU MHz: 1397.031
CPU max MHz: 5000.6831
CPU min MHz: 1400.0000
BogoMIPS: 7784.71
CPU cache size: 512 KB
No brand USB OPTICAL MOUSE
Microsoft Corp. Microsoft Ergonomic Keyboard
Filesystem Size Used Avail Use% Mounted on
udev 14G 0 14G 0% /dev
tmpfs 2.8G 1.5M 2.8G 1% /run
/dev/nvme0n1p2 233G 22G 199G 10% /
tmpfs 14G 0 14G 0% /dev/shm
tmpfs 5.0M 4.0K 5.0M 1% /run/lock
/dev/nvme0n1p1 511M 3.5M 508M 1% /boot/efi
/dev/sdb1 3.6T 93G 3.4T 3% /fourTB
/dev/sda1 5.5T 2.3T 2.9T 45% /sixTB
tmpfs 2.8G 132K 2.8G 1% /run/user/1000
FWIW, in previous versions of gdb, I could always print a cosine or other math function.
OK, the above comment's solution worked once, and then quit. cos(.333) aborted gdb. Oh well... I'm wondering if it's a gdb or Debian problem, or that my machine's hardware is simply weird. I also neglected to include in the above comment's command "install" The command should read:
apg-get install gdb gdb-doc build-essential devscript

why is my linux system experiencing "Log I/O error Detected. Shutting down filesystem" problem

system info:
[root#cpe ~]# uname -a
Linux cpe 3.10.0-514.26.2.el7.x86_64 #1 SMP Tue Jul 4 15:04:05 UTC 2017 x86_64 x86_64 x86_64 GNU/Linux
[root#cpe ~]# cat /etc/redhat-release
CentOS Linux release 7.3.1611 (Core)
[root#cpe ~]# df
Filesystem 1K-blocks Used Available Use% Mounted on
/dev/mapper/cl_cpe-root 34721216 34721196 20 100% /
devtmpfs 32843124 0 32843124 0% /dev
tmpfs 32855080 0 32855080 0% /dev/shm
tmpfs 32855080 942644 31912436 3% /run
tmpfs 32855080 0 32855080 0% /sys/fs/cgroup
/dev/mapper/cl_cpe-home 16947200 32944 16914256 1% /home
/dev/sda1 1038336 85484 952852 9% /boot
tmpfs 6571016 0 6571016 0% /run/user/0
[root#cpe ~]# mount | grep root
/dev/mapper/cl_cpe-root on / type xfs (rw,relatime,attr2,inode64,noquota)
problem:
when system is running, there are error log as below. then can't run any command line.
XFS (dm-0): metadata I/O error: block 0x2128c70 ("xlog_iodone") error 5 numblks 64
XFS (dm-0): Log I/O Error Detected. Shutting down filesystem
XFS (dm-0): Please umount the filesystem and rectify the problems(s)
XFS (dm-0): metadata I/O error: block 0x2128c7f ("xlog_iodone") error 5 numblks 64
XFS (dm-0): metadata I/O error: block 0x2128c82 ("xlog_iodone") error 5 numblks 64
Who can help me to analyze or locate the problem.
Thanks in advance.

Logwatch is too noisy

I've been using Logwatch for at least 12 years, but since I've moved to Ubuntu 18.04 I've gotten soooo annoyed about the daily e-mail is listing 37 /snap in the filesystem check:
Filesystem Size Used Avail Use% Mounted on
/dev/nvme0n1p2 439G 268G 149G 65% /
/dev/loop0 83M 83M 0 100% /snap/shotcut/119
/dev/loop1 234M 234M 0 100% /snap/gimp/322
/dev/loop3 291M 291M 0 100% /snap/vlc/1620
/dev/loop4 218M 218M 0 100% /snap/gnome-3-34-1804/60
/dev/loop2 256K 256K 0 100% /snap/gtk2-common-themes/13
etc...
I have looked for a solution before without luck and I've been looking in the logwatch files, I couldn't find any settings to do this.
I looked in /usr/share/logwatch/scripts/services/zz-disk_space, where the df command is:
df -h -x tmpfs -x devtmpfs -x udf -x iso9660
Filesystem Size Used Avail Use% Mounted on
/dev/nvme0n1p2 439G 268G 150G 65% /
/dev/loop0 83M 83M 0 100% /snap/shotcut/119
/dev/loop1 234M 234M 0 100% /snap/gimp/322
/dev/loop3 291M 291M 0 100% /snap/vlc/1620
etc... (37 of those in total)
By adding '-x squashfs' i get what i want:
df -h -x tmpfs -x devtmpfs -x udf -x iso9660 -x squashfs
Filesystem Size Used Avail Use% Mounted on
/dev/nvme0n1p2 439G 268G 150G 65% /
/dev/sda 3.6T 580G 2.9T 17% /backup
/dev/nvme0n1p1 511M 7.4M 504M 2% /boot/efi
//192.168.0.200/nas-office/backup 1.9T 723G 1.2T 39% /mnt/nas
Excellent!

Mounting ext4 mounts as ext2 in linux 4.4.0 [RHEL]

commands I executed
mkfs .ext4 -F /dev/xxx0
mke2fs 1.42.9 (28-Dec-2013)
Filesystem label=
OS type: Linux
Block size=4096 (log=2)
Fragment size=4096 (log=2)
Stride=0 blocks, Stripe width=0 blocks
524288 inodes, 2097152 blocks
104857 blocks (5.00%) reserved for the super user
First data block=0
Maximum filesystem blocks=2147483648
64 block groups
32768 blocks per group, 32768 fragments per group
8192 inodes per group
Superblock backups stored on blocks:
32768, 98304, 163840, 229376, 294912, 819200, 884736, 1605632
Allocating group tables: done
Writing inode tables: done
Writing superblocks and filesystem accounting information: done
mount -o dax /dev/xxx0 /mnt/ext4-xxx0
mount output
/dev/xxx0 on /mnt/ext4-xxx0 type ext2 (rw,relatime,seclabel,block_validity,barrier,dax,user_xattr,acl)
dmesg output
[ 2917.044866] EXT4-fs (xxx0): DAX enabled. Warning: EXPERIMENTAL, use at your own risk
[ 2917.044869] EXT4-fs (xxx0): mounting ext2 file system using the ext4 subsystem
...
...
[ 2917.045459] EXT4-fs (xxx0): mounted filesystem without journal. Opts: dax
which subsequently results in failure of fallocate, as the ext2 is not supported fos ext2.
any suggestions/solution would be appreciated.
Provide the type as ext4 while mounting. Or else it will take default type (or) type mentioned in /etc/fstab.
# mount -o dax -t ext4 /dev/xxx0 /mnt/ext4-xxx0

if there is no sysfs in /etc/fstab, how sysfs is mounted?

If I type $mount on shell, it shows several mount point
proc on /proc type proc (rw,nosuid,nodev,noexec,relatime)
sys on /sys type sysfs (rw,nosuid,nodev,noexec,relatime)
dev on /dev type devtmpfs (rw,nosuid,relatime,size=3030352k,nr_inodes=757588,mode=755)
....
....
tmpfs on /run/user/1000 type tmpfs (rw,nosuid,nodev,relatime,size=606608k,mode=700,uid=1000,gid=1000)
binfmt_misc on /proc/sys/fs/binfmt_misc type binfmt_misc (rw,relatime)
I searched web and it says i could setup mount point by editing fatab.(like sysfs /sys sysfs noauto 0 0)
but in my /etc/fstab, there is no such description about sysfs, it only has descrption about /dev/sda1.
Then, how does my linux understand sysfs mount point? and who mounts it?
FYI:
Does sysfs procfs devtmpfs mounted by kernel?
systemd tries to mount, but earlier than that, initramfs mount it.

Resources