Solaris Kernel Memory

From time to time I see some solaris server using a lot of memory "just" for the kernel. You could take a deeper look to see where the kernel memory is used:

root@server:~# mdb -k
Loading modules: [ unix genunix specfs dtrace zfs scsi_vhci sd mpt_sas mac px ldc ip hook neti ds arp usba kssl stmf stmf_sbd random sockfs md niumx idm
cpc crypto fcip fctl fcp mdesc vldc smbsrv nfs zvmm ufs logindmux ptm ii nsctl sppp nsmb rdc sdbc sv lofs ipc ]

>
> ::kmastat
cache                        buf    buf    buf    memory     alloc alloc
name                        size in use  total    in use   succeed  fail
------------------------- ------ ------ ------ ---------- --------- -----
kmem_magazine_1               16  13787  39546     638976B   2212036     0
kmem_magazine_3               32  13293 149017    4825088B  20882637     0
kmem_magazine_7               64  63818 687708   44711936B  74367950     0
kmem_magazine_15             128  81728 154728   20119552B  40689829     0
kmem_magazine_31             256      0      0          0B         0     0
kmem_magazine_47             384      0      0          0B         0     0
kmem_magazine_63             512      0      0          0B         0     0
kmem_magazine_95             768      0      0          0B         0     0
kmem_magazine_127           1024      0      0          0B         0     0
kmem_magazine_143           1152      0      0          0B         0     0
kmem_magazine_179           1440      0      0          0B         0     0
kmem_magazine_255           2048      0      0          0B         0     0
kmem_magazine_361           2896      0      0          0B         0     0
kmem_magazine_492           3944      0      0          0B         0     0
kmem_slab_cache               72  21964  41888    3063808B   1905292     0
kmem_bufctl_cache             24  60474  84162    2039808B   1966131     0
kmem_bufctl_audit_cache      128      0      0          0B         0     0
kmem_va_8192                8192   6617  15040  123207680B     92929     0
kmem_va_16384              16384     12     32     524288B        22     0
kmem_va_24576              24576     11     50    1310720B       147     0
kmem_va_32768              32768      6     64    2097152B      1152     0
kmem_va_40960              40960      9    192    8388608B      4001     0
kmem_va_49152              49152      9    200   10485760B      7542     0
kmem_va_57344              57344      1     48    3145728B      1143     0
kmem_va_65536              65536     18    144    9437184B      4846     0
kmem_alloc_8                   8 225785 229390    1851392B 1460654149     0
kmem_alloc_16                 16  70050  74022    1196032B 1802895553     0
kmem_alloc_24                 24 176486 244712    5931008B 4193514113     0
kmem_alloc_32                 32  40268  45034    1458176B 3716814702     0
kmem_alloc_40                 40  31285  47705    1925120B 1528348140     0
kmem_alloc_48                 48  67277 128102    6209536B 1790577690     0
kmem_alloc_56                 56  91174 141955    8019968B 858181106     0
kmem_alloc_64                 64 151639 323946   21061632B 1933863414     0
kmem_alloc_80                 80 187667 261994   21250048B 596556789     0
kmem_alloc_96                 96   9275  29064    2834432B 1763944514     0
kmem_alloc_112               112   8553  16056    1826816B 242740674     0
kmem_alloc_128               128 108755 111636   14516224B 241660346     0
kmem_alloc_160               160   4200  18000    2949120B 378366571     0
kmem_alloc_192               192  60314 130536   25460736B 516119668     0
kmem_alloc_224               224   1682   2232     507904B 498796577     0
kmem_alloc_256               256   2237  51305   13557760B 1199808091     0
kmem_alloc_320               320   4686  18475    6053888B 4005106805     0
kmem_alloc_384               384    664   2289     892928B 1743958062     0
kmem_alloc_448               448   1487   1890     860160B 156543851     0
kmem_alloc_512               512   6052   6645    3629056B 283536972     0
kmem_alloc_640               640  48054  63216   43155456B  49035869     0
kmem_alloc_768               768    181    510     417792B 533346028     0
kmem_alloc_896               896    106    423     385024B   2201063     0
kmem_alloc_1152             1152   3164   3563    4169728B 758869183     0
kmem_alloc_1344             1344    449    768    1048576B 134708639     0
kmem_alloc_1600             1600    130    490     802816B  12434972     0
kmem_alloc_2048             2048    862   1080    2211840B  49268582     0
kmem_alloc_2688             2688    147    471    1286144B 228164685     0
kmem_alloc_4096             4096   2311   2738   11214848B 599289299     0
kmem_alloc_8192             8192   6843   7355   60252160B 823767889     0
kmem_alloc_12288           12288     24    286    3514368B  94265288     0
kmem_alloc_16384           16384    448   4864   79691776B 372203661     0
kmem_alloc_24576           24576    247    358    8798208B  20182582     0
kmem_alloc_32768           32768    625    829   27164672B  67372957     0
kmem_alloc_40960           40960    292    497   20357120B  10594602     0
kmem_alloc_49152           49152    264    424   20840448B  18226119     0
kmem_alloc_57344           57344     57    140    8028160B   7752775     0
kmem_alloc_65536           65536     83    146    9568256B  68801457     0
kmem_alloc_73728           73728    254    318   23445504B   1224400     0
kmem_alloc_81920           81920     12     75    6144000B    855948     0
kmem_alloc_90112           90112     16     65    5857280B    772093     0
kmem_alloc_98304           98304     72    130   12779520B    673798     0
kmem_alloc_106496         106496      5     60    6389760B    649065     0
kmem_alloc_114688         114688      9     63    7225344B    796342     0
kmem_alloc_122880         122880      3     59    7249920B    388702     0
kmem_alloc_131072         131072     20    352   46137344B  46679183     0
kmem_alloc_262144         262144     25     89   23330816B     22305     0
kmem_alloc_524288         524288     10     91   47710208B    292417     0
kmem_alloc_1048576        1048576     14     83   87031808B    151526     0
streams_mblk                  64  11628  17892    1163264B 4222106588     0
streams_dblk_64              192   1880   3948     770048B 1560260262     0
streams_dblk_128             256      6    744     196608B 2094673871     0
streams_dblk_192             320      2    500     163840B 407759363     0
streams_dblk_256             384    206   1911     745472B 384014686     0
streams_dblk_320             448      0    360     163840B 916550415     0
streams_dblk_512             640      0    312     212992B 122220096     0
streams_dblk_1024           1152      0    322     376832B  60312175     0
streams_dblk_1536           1664      2    468     851968B 4237545508     0
streams_dblk_1920           2048      0    180     368640B   2582208     0
streams_dblk_2560           2688      0    276     753664B   9535544     0
streams_dblk_4096           4224      0     81     368640B  10214770     0
streams_dblk_8192            128      0    693      90112B 319193334     0
streams_dblk_12288         12416      0     98    1261568B  17628988     0
streams_dblk_16384           128      0    315      40960B   2089403     0
streams_dblk_20480         20608      0     56    1179648B   9157299     0
streams_dblk_24576           128      0    315      40960B     23840     0
streams_dblk_28672         28800      0     28     819200B     37109     0
streams_dblk_32768           128      0    567      73728B     72012     0
streams_dblk_36864         36992      0     56    2097152B   1752035     0
streams_dblk_40960           128      0    378      49152B     89538     0
streams_dblk_45056         45184      0     21     958464B    122283     0
streams_dblk_49152           128      0    378      49152B     72414     0
streams_dblk_53248         53376      0     49    2637824B  36207829     0
streams_dblk_57344           128      0     63       8192B        80     0
streams_dblk_61440         61568      0     21    1302528B     30352     0
streams_dblk_65536           128      0     63       8192B       136     0
streams_dblk_69632         69760      0      0          0B         0     0
streams_dblk_73728           128      0      0          0B         0     0
streams_dblk_esb             128   2048   2709     352256B 310293725     0
streams_dblk_mdc             128      0      0          0B         0     0
streams_fthdr                408      0      0          0B         0     0
streams_ftblk                376      0      0          0B         0     0
multidata                    248      0      0          0B         0     0
multidata_pdslab            7112      0      0          0B         0     0
multidata_pattbl              32      0      0          0B         0     0
log_cons_cache                48     29   1352      65536B    943715     0
taskq_ent_cache               56  17219  36685    2072576B 105416892     0
taskq_cache                  280    307    348      98304B       417     0
id32_cache                    32      7    759      24576B  61202888     0
One_wallet_cache              68   3257   4480     327680B  93172593     0
Pac_nopredictor_pool      505536      1      7    3538944B         1     0
Mo_cache                     288      0      0          0B         0     0
Monode_prealloc_cache        104      0      0          0B         0     0
Mo_policy_cache               72      0      0          0B         0     0
Mo_resident_cache             72      0      0          0B         0     0
Mo_capture_cache             376      0      0          0B         0     0
Mo_caphead_cache              64      0      0          0B         0     0
Mw_later_cache               128      0      0          0B         0     0
Mw_cache                     128      0      0          0B         0     0
fakemw                        88      0    460      40960B 306059268     0
mvec_tracking                  8      0   2030      16384B 1104075934     0
mvec_tag                      48      0      0          0B         0     0
Memseg_cache                  64      0      0          0B         0     0
bp_map_8192                 8192      0      0          0B         0     0
bp_map_16384               16384      0     80    1310720B      1823     0
bp_map_24576               24576      0     80    2097152B      2443     0
bp_map_32768               32768      0     80    2621440B      3048     0
bp_map_40960               40960      0      0          0B         0     0
bp_map_49152               49152      0      0          0B         0     0
bp_map_57344               57344      0      0          0B         0     0
bp_map_65536               65536      0      0          0B         0     0
mod_hash_entries              24   1635   3042      73728B  11631688     0
ipp_mod                      304      0      0          0B         0     0
ipp_action                   368      0      0          0B         0     0
ipp_packet                    64      0      0          0B         0     0
mmuctxdom_cache              696      8     11       8192B         8     0
sfmmuid_cache               1176    370    636     868352B 132755208     0
sfmmu_tsbinfo_cache           64    384   1638     106496B 277806748     0
sfmmu_tsb8k_cache           8192      0      0          0B         0     0
sfmmu_tsb_cache             8192    104    218    1785856B 130478770     0
sfmmu8_cache                 320 359954 380900  124813312B 230714082     0
sfmmu1_cache                  96 1160768 1277472  124583936B 975785584     0
pa_hment_cache                64    384   1764     114688B 152854640     0
ism_blk_cache                336      0      0          0B         0     0
ism_ment_cache                32      0      0          0B         0     0
srd_cache                   2192    120    407     909312B  36959896     0
region_cache                 144    186    840     122880B  38085849     0
scd_cache                   2192      0      0          0B         0     0
seg_cache                    112  27575  33264    3784704B 2966412882     0
seg_pcache                   104      0    624      65536B     52128     0
vfs_cache                    240    100    561     139264B     24676     0
vn_cache                     216  99646 140492   37126144B 346774691     0
shadow_cache                  72      0      0          0B         0     0
vsk_anchor_cache              40     39    203       8192B       138     0
nep_cache                    384      5    294     114688B      1429     0
dev_info_node_cache          760    283    310     253952B       830     0
ndi_fm_entry_cache            32   5392   6831     221184B 170484658     0
kcf_sreq_cache                56      0    126       8192B   1542463     0
kcf_areq_cache               296      0     25       8192B        25     0
kcf_context_cache            112      0      0          0B         0     0
object_handle                 80 2798905 3152109  255664128B 931834012     0
object_debug_handle          216      0      0          0B         0     0
object_event                  40      0      0          0B         0     0
segkmem_ppa_262144        262144      0     12    3145728B        16     0
segkp_8192                  8192    375    512    4194304B  73688442     0
segkp_16384                16384      0      0          0B         0     0
segkp_24576                24576      0      0          0B         0     0
segkp_32768                32768      0      0          0B         0     0
segkp_40960                40960   3323   3363  146931712B    507168     0
umem_np_8192                8192      0    128    1048576B    555389     0
umem_np_16384              16384      0     80    1310720B     24633     0
umem_np_24576              24576      0      0          0B         0     0
umem_np_32768              32768      0    104    3407872B    484312     0
umem_np_40960              40960      0     90    3932160B    459456     0
umem_np_49152              49152      0      0          0B         0     0
umem_np_57344              57344      0      0          0B         0     0
umem_np_65536              65536      0     68    4456448B     24633     0
thread_cache                1040    915   1545    1687552B 107833858     0
wbuf32_cache                 512    741    900     491520B  82343977     0
wbuf64_cache                1024    775   1141    1335296B   1121736     0
lwp_cache                   1048   1516   1785    1949696B   5360544     0
turnstile_cache               64   3245   4788     311296B  87013637     0
rw_reentrd_cache             136   3245   4130     573440B  93172717     0
tslabel_cache                 48      2    169       8192B         2     0
cred_cache                   184   1137   3520     655360B 111662850     0
proc_ac_cache                 64    301   1512      98304B  81421934     0
rctl_cache                    48   6392   8957     434176B 1288617008     0
rctl_val_cache                64  13614  18018    1171456B 2940692401     0
task_cache                   160    160    800     131072B   1083893     0
kmem_defrag_cache            224      2     36       8192B         2     0
kmem_move_cache               56      0  18705    1056768B   6862115     0
i_dmahdl                    2648      0      0          0B         0     0
timeout_request              128      0      0          0B         0     0
cyclic_id_cache               80    263    303      24576B       329     0
callout_cachebabecafe         80   3239   3276     425984B      3239     0
callout_lcachebabecafe        48  10163  10206     663552B     10163     0
bounds_predictor          505536      3      7    3538944B         3     0
dnlc_space_cache              24      0      0          0B         0     0
file_cache                    72   5236   7392     540672B 2852162077     0
stream_head_cache            376    518    798     311296B 122978665     0
queue_cache                  664   1176   1536    1048576B 128578727     0
syncq_cache                  168     81    432      73728B     20512     0
qband_cache                   64      2    126       8192B         2     0
linkinfo_cache                48     50    507      24576B      5064     0
ciputctrl_cache             1024      0      0          0B         0     0
serializer_cache              64     55   1008      65536B    163607     0
as_cache                     352    371    667     237568B 132754981     0
marker_cache                 128      0    378      49152B    715850     0
anon_cache                    48 164399 212771   10313728B 2491485009     0
anonmap_cache                120  15546  19363    2367488B 3655840909     0
segvn_cache                  224  27575  32724    7446528B 2756160318     0
segvn_szc_cache1              64      0    882      57344B 772856386     0
segvn_szc_cache2             512      0      0          0B         0     0
segvn_szc_cache3            4096      0     88     360448B  35897056     0
segvn_szc_cache4           32768      0      0          0B         0     0
segvn_szc_cache5          262144      0      0          0B         0     0
segvn_szc_cache6          2097152      0      0          0B         0     0
flk_edges                     48      0    169       8192B       409     0
fdb_cache                    104      0      0          0B         0     0
timer_cache                  176      4     46       8192B        31     0
vmu_bound_cache               56   6650   7540     425984B     15900     0
vmu_object_cache              88   2305   2484     221184B      3234     0
physio_buf_cache             248      0    416     106496B     74677     0
process_cache               4168    383    468    2129920B  92796816     0
numaio_obj_cache             328    112    192      65536B       425     0
numaio_grp_cache             144      9     56       8192B        27     0
mac_impl_cache             13488      6      9     122880B         8     0
mac_ring_cache               480      8     30      16384B        12     0
mac_block_cache              152      0      0          0B         0     0
mac_descriptor_cache          64      0      0          0B         0     0
mac_packet_pool_cache       1184      0      0          0B         0     0
mac_magazine_cache           552      0      0          0B         0     0
flow_tab_cache_0             184      5     42       8192B         7     0
flow_entry_cache_0         22440     12     20     450560B        26     0
mac_bcast_grp_cache           80      7    101       8192B        19     0
mac_client_impl_cache       2064      6     11      24576B         8     0
mac_promisc_impl_cache       120      0      0          0B         0     0
ip_minor_arena_sa_1            1     46    384        384B   1721432     0
ip_minor_arena_la_1            1     65   1088       1088B    959164     0
ip_conn_cache                744      3     60      49152B      1193     0
tcp_conn_cache              2120    314    693    1548288B    737336     0
udp_conn_cache              1256     69    324     442368B   2592397     0
rawip_conn_cache            1096      0    140     163840B     17159     0
rts_conn_cache               816      8     27      24576B        30     0
ire_cache                    352    153    168      65536B       202     0
ncec_cache                   200    110    279      73728B      6158     0
nce_cache                    112    117    378      49152B      6454     0
rt_entry                     152    133    168      32768B       180     0
radix_mask                    32      8    253       8192B        15     0
radix_node                   120      5     67       8192B         5     0
ipsec_actions                 88      0      0          0B         0     0
ipsec_selectors               80      0      0          0B         0     0
ipsec_policy                  80      0      0          0B         0     0
tcp_timercache                88   1239   2024     180224B    726650     0
tcp_notsack_blk_cache         24      1   1690      40960B   2083719     0
squeue_cache                 168    100    126      24576B       100     0
sctp_conn_cache             2608      0      0          0B         0     0
sctp_faddr_cache             472      0      0          0B         0     0
sctp_set_cache                24      0      0          0B         0     0
sctp_ftsn_set_cache           16      0      0          0B         0     0
dce_cache                    152    179    265      40960B       239     0
ire_gw_secattr_cache          24      0      0          0B         0     0
ldc_memhdl_cache              48      0      0          0B         0     0
ldc_memseg_cache              64      0      0          0B         0     0
fnode_cache                  176      9     84      16384B    121213     0
pipe_cache                   320     66    350     114688B  57668150     0
snode_cache                  152    858   1590     245760B 208309593     0
clnt_clts_endpnt_cache        88      0      0          0B         0     0
bpmap_cache                  200      0      0          0B         0     0
zio_cache                    912     26  76848   78692352B 2230982187     0
zio_link_cache                48      0  82472    3997696B 3836987868     0
sa_cache                      56  87949 132675    7495680B 197787650     0
dnode_t                      696 954000 954371  710746112B 104337805     0
dmu_buf_impl_t               216 1769940 2097530  464404480B 206659141     0
arc_elink_t                   32 967948 1389982   45006848B 3414386709     0
arc_buf_t                    168 992393 1401744  239230976B 559040333     0
arc_ref_t                     72 1769981 2097760  153436160B  54845411     0
arc_ghost_t                   64 314633 317646   20652032B 208588997     0
arc_meta                     184 956283 1262932  235134976B  57748154     0
arc_data                     184 915092 924264  172081152B 109836972     0
arc_data_512                  16 743953 756951   12230656B 122178970     0
arc_meta_512                  16 843315 1134159   18325504B  82077912     0
arc_data_1024                 16   3656  12168     196608B   1870065     0
arc_meta_1024                 16    380   2535      40960B    291201     0
arc_data_1536                 16   2409   8112     131072B   1191893     0
arc_meta_1536                 16     72   1521      24576B     43830     0
arc_data_2048                 16   1994   8112     131072B   1125267     0
arc_meta_2048                 16    307   2028      32768B     76657     0
arc_data_3072                 16   2210  11154     180224B   3490145     0
arc_meta_3072                 16     50   1521      24576B     20353     0
arc_data_4096                 16   1369  15210     245760B   5126190     0
arc_meta_4096                 16   5662   7098     114688B   1601065     0
arc_data_6144                 16   1864  10647     172032B   2382865     0
arc_meta_6144                 16     20   1521      24576B     40232     0
arc_data_8192                 16    998   7605     122880B   1823731     0
arc_meta_8192                 16     15   1014      16384B      3434     0
arc_data_12288                16    958   7605     122880B   1551591     0
arc_meta_12288                16     32   1014      16384B      8738     0
arc_data_16384                16    513   5577      90112B   1210128     0
arc_meta_16384                16  90237 305214    4931584B  15250828     0
arc_data_24576                16    458  20787     335872B  10999937     0
arc_meta_24576                16     19   1014      16384B      5337     0
arc_data_32768                16    428  25857     417792B   7619052     0
arc_meta_32768                16     10   1014      16384B      2976     0
arc_data_40960                16    163  22308     360448B  15904627     0
arc_meta_40960                16     13   1521      24576B     22600     0
arc_data_49152                16    227  19266     311296B   7580554     0
arc_meta_49152                16      1    507       8192B       205     0
arc_data_57344                16   2956  13182     212992B   3697718     0
arc_meta_57344                16      0    507       8192B       175     0
arc_data_65536                16   1638   3042      49152B    967486     0
arc_meta_65536                16      0    507       8192B       359     0
arc_data_73728                16     94   2028      32768B    569502     0
arc_meta_73728                16      4    507       8192B       356     0
arc_data_81920                16    448   1521      24576B    528592     0
arc_meta_81920                16      3    507       8192B       542     0
arc_data_90112                16   1492   2535      40960B    675906     0
arc_meta_90112                16      1    507       8192B       204     0
arc_data_98304                16     68   1521      24576B    428205     0
arc_meta_98304                16      1    507       8192B       151     0
arc_data_106496               16     47   1521      24576B    381247     0
arc_meta_106496               16      1    507       8192B       248     0
arc_data_114688               16     32   1521      24576B    387053     0
arc_meta_114688               16      1   1014      16384B      5162     0
arc_data_122880               16     23   1521      24576B    396002     0
arc_meta_122880               16      1    507       8192B       263     0
arc_data_131072               16 126094 142467    2301952B 3965382018     0
arc_meta_131072               16     29   1521      24576B    316793     0
arc_data_139264               16      0      0          0B         0     0
arc_meta_139264               16      0      0          0B         0     0
arc_data_262144               16      0      0          0B         0     0
arc_meta_262144               16      0      0          0B         0     0
arc_data_524288               16      0      0          0B         0     0
arc_meta_524288               16      0      0          0B         0     0
arc_data_1048576              16      0      0          0B         0     0
arc_meta_1048576              16      0      0          0B         0     0
l2arc_seg_t                   96      0      0          0B         0     0
l2arc_buf_t                   80      0      0          0B         0     0
zfetch_trigger_t              80    189   2525     204800B  61215486     0
space_seg_cache               64  35260 482076   31342592B 2637153477     0
dsl_share_t                  328      1     24       8192B         1     0
dsl_share_state_t             48      0      0          0B         0     0
zil_lwb_cache                208      6    624     131072B    736315     0
zil_train_cache               64      2   1386      90112B   2295082     0
zil_car_cache                 56      2   1885     106496B   3394521     0
zil_ian_cache                 80     32   1818     147456B  63267405     0
vdev_disk_cache              256      0    341      90112B 882654057     0
zfs_znode_cache              328  87949  91200   31129600B 243237064     0
dls_link_cache               344      6     23       8192B         9     0
dls_devnet_cache             368      6     22       8192B         8     0
px0_px0_0_cache1            8192     12     32     262144B 112535534     0
px0_px0_0_cache2           16384      3      8     131072B      9249     0
px0_px0_0_cache8           65536      2      2     131072B         2     0
dv_node_cache                184    664    748     139264B      1211     0
px0_mpt_sas0_2_cache1       8192      2     32     262144B 296721559     0
px0_mpt_sas0_2_cache2      16384      0     16     262144B  99973488     0
pkt_cache_mpt_sas_0          720      0    110      81920B 290898182     0
px0_mpt_sas1_3_cache1       8192      4     48     393216B 737561670     0
px0_mpt_sas1_3_cache2      16384      0     32     524288B 281684410     0
pkt_cache_mpt_sas_1          720      2    132      98304B 763554977     0
sdev_node_cache              248    905    992     253952B     51623     0
audit_proc                    48    385   1521      73728B  80880005     0
drv_secobj_cache             296      0      0          0B         0     0
dld_str_cache                320     11    150      49152B    147472     0
exacct_object_cache           40      0      0          0B         0     0
rw_numa_cache                128  19911  20853    2711552B   2551080     0
kssl_cache                  1624      0      0          0B         0     0
stmf_task_event_cache         64      0      0          0B         0     0
stmf_task_cache             3296      0      0          0B         0     0
stmf_ref_node_cache           16      2    507       8192B         6     0
sbd_task_cache              1304      0      0          0B         0     0
namefs_inodes_1                1     47   1152       1152B     74469     0
port_cache                    80     13    101       8192B        51     0
socket_cache                 792    321    720     589824B   1700687     0
socktpi_cache               1096      0      7       8192B         6     0
socktpi_unix_cache          1096     32    350     409600B    170699     0
sock_sod_cache               656      0      0          0B         0     0
tl_cache                     448     78    396     180224B    171292     0
keysock_1                      1      0     64         64B         1     0
spdsock_1                      1      0     64         64B         5     0
rds_alloc_cache               88      0      0          0B         0     0
dtrace_state_cache        262144      0     14    3670016B        29     0
idm_buf_cache                256      0      0          0B         0     0
idm_task_cache              1408      0      0          0B         0     0
idm_tx_pdu_cache             400      0      0          0B         0     0
idm_rx_pdu_cache             596      0      0          0B         0     0
softmac_cache                568      5     14       8192B         7     0
softmac_upper_cache          232      0      0          0B         0     0
fctl_cache                   112      0      0          0B         0     0
vldc_cookie_buf_cache     262144      0     65   17039360B  33963567     0
authkern_cache                72      0    784      57344B 205205946     0
authnone_cache                72      0      0          0B         0     0
authloopback_cache            72      0      0          0B         0     0
authdes_cache_handle          80      0      0          0B         0     0
rnode_cache                  680    982    990     737280B     61171     0
nfs_access_cache              56    182   3190     180224B    145036     0
client_handle_cache           32     17    506      16384B      1850     0
rnode4_cache                1032      0      0          0B         0     0
svnode_cache                  40      0      0          0B         0     0
nfs4_access_cache             56      0      0          0B         0     0
client_handle4_cache          32      0      0          0B         0     0
nfs4_ace4vals_cache           48      0      0          0B         0     0
nfs4_ace4_list_cache         264      0      0          0B         0     0
NFS_idmap_cache               56      0      0          0B         0     0
lm_xprt_10003c62cf40          32      0      0          0B         0     0
lm_vnode_10003c62cf40        184      0      0          0B         0     0
lm_sysid_10003c62cf40        160      0     50       8192B         1     0
lm_client_10003c62cf40       128      0     63       8192B         1     0
lm_async_10003c62cf40         32      0      0          0B         0     0
lm_sleep_10003c62cf40         96      0      0          0B         0     0
lm_config_10003c62cf40        80      2    101       8192B         2     0
uvfs_uvnode_cache            392      0      0          0B         0     0
uvfs_task_sync_cache          16      0      0          0B         0     0
uvfs_task_rootvp_cache        16      0      0          0B         0     0
uvfsvfs_cache                280      0      0          0B         0     0
ufs_inode_cache              368      0      0          0B         0     0
directio_buf_cache           272      0      0          0B         0     0
lufs_save                     24      0      0          0B         0     0
lufs_bufs                    256      0      0          0B         0     0
lufs_mapentry_cache          112      0      0          0B         0     0
pty_map                       64     54    756      49152B     10729     0
sppptun_map                  440      0      0          0B         0     0
Hex0x100030bf3428_minor_1      1      0      0          0B         0     0
Hex0x100030bf3430_minor_1      1      0      0          0B         0     0
px0_igb3_4_cache1           8192   1793   1840   15073280B 169264446     0
px0_igb3_4_cache2          16384      1     24     393216B 194088248     0
px0_igb0_5_cache1           8192   1793   2000   16384000B 3804956026     0
px0_igb0_5_cache2          16384      1     24     393216B 653588623     0
iscsit_status_pdu_cache      400      0      0          0B         0     0
stp_2_0_987                 1712      0      0          0B         0     0
stp_m2_0_987                  56      0      0          0B         0     0
audit_buffer                 152      0    212      32768B       690     0
lnode_cache                   32     12   1265      40960B 247872400     0
flow_tab_cache_1             184      1     42       8192B         1     0
flow_entry_cache_1         22440      1      4      90112B         1     0
lm_xprt_10003c62fa40          32      0      0          0B         0     0
lm_vnode_10003c62fa40        184      0      0          0B         0     0
lm_sysid_10003c62fa40        160      0      0          0B         0     0
lm_client_10003c62fa40       128      0      0          0B         0     0
lm_async_10003c62fa40         32      0      0          0B         0     0
lm_sleep_10003c62fa40         96      0      0          0B         0     0
lm_config_10003c62fa40        80      1    101       8192B         1     0
vnic_cache                  1544      1      5       8192B         1     0
crypto_session_cache         104      0      0          0B         0     0
sdp_generic_table             32      0      0          0B         0     0
sdp_advt_cache                80      0      0          0B         0     0
sdp_advt_table                24      0      0          0B         0     0
sdp_conn_cache              1944      0      0          0B         0     0
Hex0x10004c7d1428_minor_1      1      0      0          0B         0     0
Hex0x10004c7d1430_minor_1      1      0      0          0B         0     0
stp_3_1_3841                1712      0      0          0B         0     0
stp_m3_1_3841                 56      0      0          0B         0     0
stp_2_1_3866                1712      0      0          0B         0     0
stp_m2_1_3866                 56      0      0          0B         0     0
fcsm_job_cache               104      0      0          0B         0     0
aggr_port_cache             1032      0      0          0B         0     0
aggr_grp_cache              1008      0      0          0B         0     0
iptun_cache                  288      0      0          0B         0     0
smb_shr_notify_cache          72      0      0          0B         0     0
smb_share_cache              168      1     48       8192B         1     0
smb_vfs_cache                 48      0      0          0B         0     0
smb_mc_cache                  96      0      0          0B         0     0
smb_uio_cache                752      0      0          0B         0     0
smb_node_cache               752      0      0          0B         0     0
smb_txreq                  66592      0      0          0B         0     0
vxlan_grp_cache              200      0      0          0B         0     0
vxlan_cache                  904      0      0          0B         0     0
zvsdir_zvnode_cache           96      0      0          0B         0     0
px0_igb1_6_cache1           8192      0     32     262144B      3584     0
px0_igb1_6_cache2          16384      0      8     131072B         1     0
px0_igb2_7_cache1           8192      0     32     262144B      3584     0
px0_igb2_7_cache2          16384      0      8     131072B         1     0
------------------------- ------ ------ ------ ---------- --------- -----
Total [hat_memload]                             124813312B 230714082     0
Total [kmem_msb]                                 76464128B 148885992     0
Total [kmem_va]                                 158597120B    111782     0
Total [kmem_default]                           3346472960B 2897004240     0
Total [bp_map]                                    6029312B      7314     0
Total [kmem_tsb_default]                          1785856B 130478770     0
Total [hat_memload1]                            124583936B 975785584     0
Total [segkmem_ppa]                               3145728B        16     0
Total [umem_np]                                  14155776B   1548423     0
Total [id32]                                        24576B  61202888     0
Total [segkp]                                   151126016B  74195610     0
Total [ip_minor_arena_sa]                             384B   1721432     0
Total [ip_minor_arena_la]                            1088B    959164     0
Total [px0_px0_0_vmem_top]                          524288B 112544785     0
Total [px0_mpt_sas0_2_vmem_top]                          524288B 396695047     0
Total [px0_mpt_sas1_3_vmem_top]                          917504B 1019246080     0
Total [namefs_inodes]                                1152B     74469     0
Total [keysock]                                        64B         1     0
Total [spdsock]                                        64B         5     0
Total [px0_igb3_4_vmem_top]                        15466496B 363352694     0
Total [px0_igb0_5_vmem_top]                        16777216B 163577353     0
Total [px0_igb1_6_vmem_top]                          393216B      3585     0
Total [px0_igb2_7_vmem_top]                          393216B      3585     0
------------------------- ------ ------ ------ ---------- --------- -----

vmem                         memory     memory    memory     alloc alloc
name                         in use      total    import   succeed  fail
------------------------- ---------- ----------- ---------- --------- -----
heap                      4418913714176B 17592186044416B         0B    445761     0
    vmem_metadata          14204928B   14417920B  14417920B      1625     0
        vmem_seg           42156032B   42156032B  42156032B      5141     0
        vmem_hash           5945344B    5955584B   5955584B       378     0
        vmem_vmem            443992B     513592B    475136B       171     0
    heap_alloc                61760B      65536B     65536B       198     0
    hat_memload           124813312B  124813312B 124813312B     17940     0
    kstat                   1317944B    1351680B   1286144B   1091062     0
    kmem_metadata          61620224B   97779712B  97779712B    115733     0
        kmem_msb           76464128B   76464128B  76464128B    130218     0
        kmem_audit                0B          0B         0B         0     0
        kmem_cache          2236176B    3907584B   3907584B       552     0
        kmem_hash            530944B     540672B    540672B      1038     0
    kmem_log                4204640B    4210688B   4210688B         6     0
    kmem_firewall_va              0B          0B         0B         0     0
        kmem_firewall             0B          0B         0B         0     0
    mod_sysfile                   8B       8192B      8192B         1     0
    kmem_oversize         20114661376B 20122435584B 20122435584B    333208     0
    kmem_va               210501632B  210501632B 210501632B     43084     0
        kmem_default      3346472960B 3925639168B 3925639168B   5790429     0
    little_endian           1162048B    1228800B   1228800B 104879936     0
    big_endian             36849813B   58744832B  58744832B  60100613     0
    bp_map                  6029312B    6029312B   6029312B      6402     0
    ksyms                   3267768B    3317760B   3317760B       418     0
    ctf                      278772B     327680B    327680B       420     0
    kmem_bigtsb                   0B          0B         0B         0     0
        kmem_bigtsb_default         0B          0B         0B         0     0
    kmem_tsb               12582912B   12582912B  12582912B       610     0
        kmem_tsb_default   10174464B   12582912B  12582912B  28176218   579
    hat_memload1          124583936B  124583936B 124583936B     15214     0
    KOM firewall                  0B          0B         0B         0     0
    segkmem_ppa             3145728B    3145728B   3145728B         3     0
    umem_np                14155776B   14155776B  14155776B     24604     0
    contig_mem_arena       30666816B  222298112B         0B 133475242     0
    contig_mem_arena_le           0B          0B         0B         0     0
    defdump_arena         1147854848B 1147854848B 1147854848B         6     0
        defdump_metadata_arena 1147854848B 1147854848B 1147854848B         6     0
lppool                    154394624B  154394624B         0B     31840 138056
heap32                     10772480B  134217728B         0B       126     0
    id32                      24576B      24576B     24576B         3     0
    module_data             8638543B    8814592B   8290304B       596     0
    promplat                      0B          0B         0B        79     0
    trapstat                      0B          0B         0B         0     0
heaptext                   33562624B  134217728B         0B        17     0
    module_text            33551680B   37863424B         0B       421     0
logminor_space                   61B     262137B         0B    942755     0
taskq_id_arena                  146B 2147483647B         0B       240     0
heap_lp                   4026531840B 4397241204736B         0B        16     0
    kmem_lp               4026531840B 4026531840B 4026531840B      7466  4150
segkp                     151420928B 2147483648B         0B      4905     0
rctl_ids                         44B      32767B         0B        44     0
zoneid_space                      1B       9998B         0B         1     0
taskid_space                    160B     999999B         0B   1048209     0
pool_ids                          0B     999998B         0B         0     0
contracts                       159B 2147483646B         0B    853997     0
regspec                     9175040B 5368709120B         0B        31     0
mac_minor_ids                   116B     130070B         0B       792     0
ip_minor_arena_sa               384B     262140B         0B         6     0
ip_minor_arena_la              1088B 4294705152B         0B        17     0
px0_px0_0_vmem_top           655360B 1878917120B         0B      2135     8
    px0_px0_0_vmem_16             0B          0B         0B      2125     0
px0_px0_0_vmem_c                  0B  268435456B         0B         0     9
px0_mpt_sas0_2_vmem_top     1441792B  939393024B         0B  41273125     8
    px0_mpt_sas0_2_vmem_16    262144B     262144B    262144B  35835132     0
px0_mpt_sas0_2_vmem_c             0B  134217728B         0B         0     9
px0_mpt_sas1_3_vmem_top     1835008B  939393024B         0B 110855090     8
    px0_mpt_sas1_3_vmem_16    262144B     262144B    262144B  94440293     0
px0_mpt_sas1_3_vmem_c             0B  134217728B         0B         0     9
lib_va_32                   7954432B 2031599616B         0B        20     0
lib_va_64                 283328512B 2251793356234752B         0B       211     0
namefs_inodes                  1152B      65536B         0B        18     0
tl_minor_space                   78B     262138B         0B    163787     0
keysock                          64B 4294967295B         0B         1     0
spdsock                          64B 4294967295B         0B         1     0
dtrace                       104429B 4294967295B         0B   1219732     0
dtrace_minor                      0B 4294967293B         0B        28     0
syseventd_channel                15B        101B         0B     24474     0
syseventd_channel                 1B          2B         0B         1     0
idm_taskid_space                  0B      65536B         0B         0     0
module_text_holesrc_2             0B    4194304B         0B         0     0
    ktext_hole_2            1862280B    4194304B         0B       156     0
module_text_holesrc_0             0B    4194304B         0B         0     0
    ktext_hole_0            2155704B    4194304B         0B        20     0
ibcm_local_sid                    0B 4294967295B         0B         0     0
ibcm_ip_sid                       0B      65535B         0B         0     0
lmsysid_space                     1B      16383B         0B         3     0
module_text_holesrc_3             0B    4194304B         0B         0     0
    ktext_hole_3            1924624B    4194304B         0B       154     0
module_text_holesrc_4             0B    4194304B         0B         0     0
    ktext_hole_4            2109924B    4194304B         0B        72     0
module_text_holesrc_1             0B    4194304B         0B         0     0
    ktext_hole_1             236992B    4194304B         0B        32     0
logdmux_minor                    34B        256B         0B      5028     0
ptms_minor                       54B        128B         0B     10703     3
sppptun_minor                     0B         16B         0B         0     0
syseventconfd_door                1B        101B         0B         1     0
syseventconfd_door                1B          2B         0B         1     0
devfsadm_event_channel            1B        101B         0B         1     0
devfsadm_event_channel            1B          2B         0B         1     0
Hex0x100030bf3428_minor           0B 4294967294B         0B         0     0
Hex0x100030bf3430_minor           0B 4294967294B         0B         0     0
px0_igb3_4_vmem_top        15597568B  939393024B         0B  12335257     8
    px0_igb3_4_vmem_16            0B          0B         0B  12334856     0
px0_igb3_4_vmem_c                 0B  134217728B         0B         0     9
px0_igb0_5_vmem_top        16908288B  939393024B         0B    362737     8
    px0_igb0_5_vmem_16            0B          0B         0B    362449     0
px0_igb0_5_vmem_c                 0B  134217728B         0B         0     9
iscsit_tsih_pool                  0B      65535B         0B         0     0
ipnet_minor_space                 6B     262141B         0B         6     0
crypto                            0B         16B         0B    136281     0
lofi_id                           0B      16383B         0B         0     0
ds_minors                         0B     262140B         0B         0     0
Hex0x10004c7d1428_minor           0B 4294967294B         0B         0     0
Hex0x10004c7d1430_minor           0B 4294967294B         0B         0     0
semids                           90B        128B         0B        90     0
mdesc_minor                       0B        256B         0B    120953     0
aggr_portids                      0B      65534B         0B         0     0
aggr_key_ids                      0B      64535B         0B         0     0
zvmm_minor_space                  0B     262142B         0B         0     0
px0_igb1_6_vmem_top          524288B  939393024B         0B       115     8
    px0_igb1_6_vmem_16            0B          0B         0B         0     0
px0_igb1_6_vmem_c                 0B  134217728B         0B         0     9
px0_igb2_7_vmem_top          524288B  939393024B         0B       115     8
    px0_igb2_7_vmem_16            0B          0B         0B         0     0
px0_igb2_7_vmem_c                 0B  134217728B         0B         0     9
msqids                            0B        128B         0B         0     0
shmids                            0B        128B         0B         0     0
------------------------- ---------- ----------- ---------- --------- -----
>
>
> ::kmastat !awk '!/Total/ {print $4 " " $1}' | sort -n | tail
76464128B kmem_msb
97779712B kmem_metadata
124583936B hat_memload1
124813312B hat_memload
210501632B kmem_va
1147854848B defdump_arena
1147854848B defdump_metadata_arena
3925639168B kmem_default
4026531840B kmem_lp
20122435584B kmem_oversize
>
>
> ::vmem ! grep kmem_oversize
000003000008a000   kmem_oversize        20114661376  20122435584    333212     0
> 000003000008a000::print vmem_t vm_kstat.vk_free.value.l
vm_kstat.vk_free.value.l = 0x50304
> 000003000008a000::print vmem_t vm_kstat.vk_alloc.value.l
vm_kstat.vk_alloc.value.l = 0x5159d

In newer solaris releases we see the ZFS buffers as own lines in mdb... that's part of the kernel memory:

> ::memstat
Page Summary                 Pages             Bytes  %Tot
----------------- ----------------  ----------------  ----
Kernel                     3129843             23.8G   76%
Defdump prealloc            140119              1.0G    3%
ZFS Metadata                292857              2.2G    7%
ZFS File Data              2251900             17.1G   55%
Anon                        158898              1.2G    4%
Exec and libs                 6226             48.6M    0%
Page cache                    9878             77.1M    0%
failed to read 'mrp_svc'; module not present
Free (cachelist)                52              416k    0%
Free (freelist)             121576            949.8M    3%
Total                      4128768             31.5G
>


What is happening there? Well... IO IO IO 🙂

root@server:~#  dtrace -n 'fbt::vmem_alloc:entry { @[args[0]->vm_name] = sum(arg1); }'
dtrace: description 'fbt::vmem_alloc:entry ' matched 1 probe
^C

  ip_minor_arena_la                                                 1
  namefs_inodes                                                     1
  ip_minor_arena_sa                                                 2
  logminor_space                                                    2
  little_endian                                                119168
  segkp                                                        188416
  big_endian                                                  1397837
  heap                                                        3145728
  kmem_oversize                                               3145728
  px0_mpt_sas0_2_vmem_16                                      5767168
  px0_mpt_sas0_2_vmem_top                                     5767168
  px0_mpt_sas1_3_vmem_16                                     15990784
  px0_mpt_sas1_3_vmem_top                                    15990784
root@server:~#
root@server:~# dtrace -n 'fbt::vmem_alloc:entry /args[0]->vm_name == "px0_mpt_sas1_3_vmem_top"/ { @[stack()] = count(); }'
dtrace: description 'fbt::vmem_alloc:entry ' matched 1 probe
^C


              genunix`vmem_xalloc+0x670
              genunix`vmem_alloc+0x21c
              px`px_dvma_pool_default_dvma_alloc+0x180
              px`px_atu_dvma_alloc+0x94
              px`px_dvma_map+0x54
              px`px_dma_bindhdl+0xbc
              genunix`ddi_dma_buf_bind_handle+0x54
              scsi`scsi_cache_bind+0x24
              scsi`scsi_cache_init_pkt+0x2d4
              scsi`scsi_init_pkt+0x4c
              scsi_vhci`vhci_bind_transport+0x9ac
              scsi_vhci`vhci_scsi_start+0x350
              sd`sd_start_cmds+0x3a4
              sd`sd_core_iostart+0x228
              sd`sd_mapblockaddr_iostart+0x210
              sd`xbuf_iostart+0x20c
              zfs`vdev_disk_strategy+0x30
              zfs`vdev_disk_io_start+0x26c
              zfs`zio_execute+0xf4
              zfs`vdev_queue_io_done+0xb4
                9

              genunix`vmem_xalloc+0x670
              genunix`vmem_alloc+0x21c
              px`px_dvma_pool_default_dvma_alloc+0x180
              px`px_atu_dvma_alloc+0x94
              px`px_dvma_map+0x54
              px`px_dma_bindhdl+0xbc
              genunix`ddi_dma_buf_bind_handle+0x54
              scsi`scsi_cache_bind+0x24
              scsi`scsi_cache_init_pkt+0x2d4
              scsi`scsi_init_pkt+0x4c
              scsi_vhci`vhci_bind_transport+0x9ac
              scsi_vhci`vhci_scsi_start+0x350
              sd`sd_start_cmds+0x3a4
              sd`sd_core_iostart+0x228
              sd`sd_mapblockaddr_iostart+0x210
              sd`xbuf_iostart+0x20c
              zfs`vdev_disk_strategy+0x30
              zfs`vdev_disk_io_start+0x26c
              zfs`zio_execute+0xf4
              zfs`vdev_raidz_io_start+0x26c
               16

              genunix`vmem_xalloc+0x670
              genunix`vmem_alloc+0x21c
              px`px_dvma_pool_default_dvma_alloc+0x180
              px`px_atu_dvma_alloc+0x94
              px`px_dvma_map+0x54
              px`px_dma_bindhdl+0xbc
              genunix`ddi_dma_buf_bind_handle+0x54
              scsi`scsi_cache_bind+0x24
              scsi`scsi_cache_init_pkt+0x2d4
              scsi`scsi_init_pkt+0x4c
              scsi_vhci`vhci_bind_transport+0x9ac
              scsi_vhci`vhci_scsi_start+0x350
              sd`sd_start_cmds+0x3a4
              sd`sd_core_iostart+0x228
              sd`sd_mapblockaddr_iostart+0x210
              sd`xbuf_iostart+0x20c
              zfs`vdev_disk_strategy+0x30
              zfs`vdev_disk_io_start+0x26c
              zfs`zio_execute+0xf4
              zfs`vdev_queue_io_done+0x94
              110
root@server:~# dtrace -n 'fbt::vmem_alloc:entry /args[0]->vm_name == "kmem_oversize"/ { @[stack()] = count(); }'
dtrace: description 'fbt::vmem_alloc:entry ' matched 1 probe
^C


              genunix`kmem_alloc+0x160
              genunix`kmem_zalloc+0x120
              autofs`auto_calldaemon+0x1d0
              autofs`auto_null_request+0x24
              autofs`unmount_tree+0x60
              autofs`unmount_zone_tree+0xc
              unix`thread_start+0x4
                1
root@server:~#

View solaris page sizes

Solaris provides different memory page sizes, on SPARC up to 16gb. It does not always make sense to use 16gb also known as huge page sizes. The Oracle database chooses the size depending on their needs:

root@server:/# pagesize -a
8192
65536
4194304
268435456
2147483648
root@server:/# ps -ef | grep smon | wc -l
      25
root@server:/# prctl -n zone.max-shm-memory -i zone global
zone: 0: global
NAME    PRIVILEGE       VALUE    FLAG   ACTION                       RECIPIENT
zone.max-shm-memory
        usage           47.5GB
        system          16.0EB    max   deny                                 -
root@server:/#
root@server:/# mdb -k
Loading modules: [ unix genunix specfs dtrace zfs scsi_vhci ldc mac ip hook neti ds arp kssl sockfs ipc random idm mdesc cpc crypto fcip fctl fcp ufs logindmux ptm sppp nsmb nfs ]
> ::memstat
Usage Type/Subtype                      Pages    Bytes  %Tot  %Tot/%Subt
---------------------------- ---------------- -------- ----- -----------
Kernel                               10657519    81.3g  6.7%
  Regular Kernel                      8204461    62.5g        5.1%/76.9%
  Defdump prealloc                    2453058    18.7g        1.5%/23.0%
ZFS                                   1986282    15.1g  1.2%
User/Anon                           142318553     1.0t 89.5%
  Regular User/Anon                  23403481   178.5g       14.7%/16.4%
  OSM                               118915072   907.2g       74.7%/83.5%
Exec and libs                          416779     3.1g  0.2%
Page Cache                             199137     1.5g  0.1%
Free (cachelist)                        46094   360.1m  0.0%
Free                                  3365972    25.6g  2.1%
Total                               158990336     1.1t  100%
>
> ::tile -s
TILE  MN  SZC   TOTAL   PCT    USER   PCT   KCAGE   PCT
           8k   86.6g 7.14%   39.9g 3.29%   46.6g 3.84%
          64k   70.5g 5.81%   70.5g 5.81%    2.1m 0.00%
           4m  109.8g 9.05%   99.2g 8.18%   10.5g 0.87%
         256m    134g 11.0%     89g 7.33%     45g 3.70%
           2g    140g 11.5%    140g 11.5%       -     -
          16g    672g 55.3%    672g 55.3%       -     -
   total         1.1t  100%    1.0t 91.5%  102.2g 8.42%
>
> ::tile -l
TILE  MN T  TOTAL  NOSZC     8k    64k     4m   256m     2g    16g   BUSY
   0   0 +    251     14     24     10     47    147      9      -      -
   1   0 +    256    133      1     25     58     39      -      -      -
   2   0 u    256    195      1     11     30     19      -      -      -
   3   0 u    256    128      -     25     20     19      -     64      -
   4   0 u    256     66      -     17     17     28      -    128      -
   5   0 u    256     66      -     13     12     37      -    128      -
   6   0 u    256    193      -     12     23     28      -      -      -
   7   0 u    127      5      -     16     17     33     56      -      -
   8   1 u    251    184      -      6     11      2     48      -      -
   9   1 u    256    128      -     10     52      2     64      -      -
   a   1 u    256    129      -     18     38      7      -     64      -
   b   1 u    256    130      1     10     53     38     24      -      -
   c   1 +    256    128      -     12     46      6      -     64      -
   d   1 u    256    129      -     11     45      7      -     64      -
   e   1 +    256    131      2     13     42      4      -     64      -
   f   1 +    127      6     21      3     12     85      -      -      -
  10   2 u    251    130      -      -      1      -     56     64      -
  11   2 +    256    128      -      -     60     12     56      -      -
  12   2 +    256     65      2      -     30     23     72     64      -
  13   2 +    256     44     20      1     25     78     24     64      -
  14   2 k      5      -      3      -      -      2      -      -      -
   total +   4852   2132     75    213    639    616    409    768      -
> ::tile -v
            TILE M   ID   MG MN PRI K T   TOTAL    FREE     OOB  CACHED  BORROW
     20600000000      0    0  0   - g +   62.7g    4.1g    3.2g   12.5m       -
                                      u   13.7g    1.4g    1.3g   12.5m       -
                                      k     49g    2.6g    1.8g       -       -
     20600008040      1    0  0   - g +     64g    2.9g    2.4g   11.7m    1.5m
                                      u   61.2g    2.2g    1.9g   11.7m       -
                                      k    2.7g  765.1m  548.7m       -    1.5m
     20600010080      2    0  0   - g u     64g  889.3m  889.3m   11.6m       -
     206000180c0      3    0  0   - g u     64g  522.7m  522.7m   13.6m       -
     20600020100      4    0  0   - g u     64g     56k     56k    680k       -
     20600028140      5    0  0   - g u     64g       -       -     32k       -
     20600030180      6    0  0   - g u     64g  537.2m  537.2m   22.0m       -
     206000381c0      7    0  0   - g u   31.7g  163.0m  159.2m   12.2m       -
     20600040200      8    1  1   - g u   62.7g    744k    744k    360k       -
     20600048240      9    1  1   - g u     64g  828.2m  828.2m    8.4m       -
     20600050280      a    1  1   - g u     64g  749.5m  749.5m   15.4m       -
     206000582c0      b    1  1   - g u     64g    1.1g  897.6m   22.2m       -
     20600060300      c    1  1   - g +     64g    1.0g    1.0g   32.4m       -
                                      u     63g    1.0g    1.0g   32.4m       -
                                      k      1g     52m       -       -       -
     20600068340      d    1  1   - g u     64g  987.1m  987.1m   15.5m       -
     20600070380      e    1  1   - g +     64g  830.3m  566.3m   26.2m       -
                                      u     62g  570.3m  566.3m   26.2m       -
                                      k      2g  260.0m       -       -       -
     206000783c0      f    1  1   - g +   31.7g    3.6g    1.1g    480k       -
                                      u      3g     88k     88k    480k       -
                                      k   28.7g    3.6g    1.1g       -       -
     20600080400     10    2  2   - g u   62.7g     16k     16k    128k       -
     20600088440     11    2  2   - g +     64g  728.2m  392.2m    1.2m   48.1m
                                      u   62.5g  604.1m  392.1m    1.2m       -
                                      k    1.5g  124.0m     64k       -   48.1m
     20600090480     12    2  2   - g +     64g  187.8m  183.8m    512k       -
                                      u     63g  135.8m  131.8m    512k       -
                                      k      1g     52m     52m       -       -
     206000984c0     13    2  2   - g +     64g    6.4g    3.3g    744k       -
                                      u     49g  116.9m  112.9m    744k       -
                                      k     15g    6.3g    3.2g       -       -
     206000a0500     14    2  2   - g k    1.2g       -       -       -       -
                                total +    1.1t   25.7g   17.7g  208.4m   49.6m
                                      u    1.0t   11.8g   10.9g  208.4m       -
                                      k  102.2g   13.9g    6.8g       -   49.6m
>