Claudeを使ってzfsのI/OバッファやARC(メモリキャッシュ)の使用率を調べるプログラムを作ってみた

UDPファイル転送プログラムの問題点を調べるために、Claudeを使ってzfsのI/OバッファやARC(メモリキャッシュ)の使用率を調べるプログラムを作ってみた。

最初に以下のプロンプトをClaudeに投げた。


Solrais11.4 for x86でzfsのI/OバッファやARC(メモリキャッシュ)の使用率を1ミリ秒ごとに出力するプログラムのソースコードをC言語を使って作成して下さい。


その回答。


まず実際のkstatフィールドを確認してから正確なコードを作成します。
ZFS kstat診断ツール

ZFS kstat診断ツールのソースコードは以下のとおり。

/*
 * zfs_kstat_dump.c
 *
 * Solaris 11.4 x86 上で ZFS / ARC 関連の kstat フィールドを
 * 全列挙する診断ツール。
 *
 * コンパイル:
 *   cc  -xc99 -o zfs_kstat_dump zfs_kstat_dump.c -lkstat
 *   gcc -std=c99 -o zfs_kstat_dump zfs_kstat_dump.c -lkstat
 *
 * 実行:
 *   pfexec ./zfs_kstat_dump 2>&1 | tee zfs_kstat_dump.log
 */
 
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <kstat.h>
#include <stdint.h>
#ifndef UINT64_MAX
#define UINT64_MAX ((uint64_t)0xFFFFFFFFFFFFFFFFULL)
#endif
 
/* -------------------------------------------------------
 * kstat の全フィールドを列挙する
 * ------------------------------------------------------- */
static void
dump_kstat(kstat_ctl_t *kc, const char *module, const char *name)
{
    kstat_t       *ksp;
    kstat_named_t *knp;
    uint_t         i;
 
    printf("\n=== kstat  module=\"%s\"  name=\"%s\" ===\n", module, name);
 
    ksp = kstat_lookup(kc, (char *)module, -1, (char *)name);
    if (ksp == NULL) {
        printf("  [NOT FOUND] errno=%d (%s)\n", errno, strerror(errno));
        return;
    }
 
    printf("  ks_type=%d  ks_ndata=%u  ks_instance=%d\n",
           ksp->ks_type, ksp->ks_ndata, ksp->ks_instance);
 
    if (kstat_read(kc, ksp, NULL) == -1) {
        printf("  [ERROR] kstat_read: %s\n", strerror(errno));
        return;
    }
 
    if (ksp->ks_type != KSTAT_TYPE_NAMED) {
        printf("  [SKIP] ks_type is not KSTAT_TYPE_NAMED (%d)\n",
               ksp->ks_type);
        return;
    }
 
    knp = (kstat_named_t *)ksp->ks_data;
    for (i = 0; i < ksp->ks_ndata; i++, knp++) {
        printf("  [%3u] %-45s type=%d  val=",
               i, knp->name, knp->data_type);
        switch (knp->data_type) {
        case KSTAT_DATA_UINT64:
            printf("%llu\n", (unsigned long long)knp->value.ui64); break;
        case KSTAT_DATA_INT64:
            printf("%lld\n", (long long)knp->value.i64);           break;
        case KSTAT_DATA_UINT32:
            printf("%lu\n",  (unsigned long)knp->value.ui32);      break;
        case KSTAT_DATA_INT32:
            printf("%ld\n",  (long)knp->value.i32);                break;
        case KSTAT_DATA_STRING:
            printf("\"%s\"\n", KSTAT_NAMED_STR_PTR(knp));          break;
        default:
            printf("(data_type=%d)\n", knp->data_type);            break;
        }
    }
}
 
/* -------------------------------------------------------
 * kstat チェーンから zfs / unix / spl 関連エントリを列挙
 * ------------------------------------------------------- */
static void
scan_chain(kstat_ctl_t *kc)
{
    kstat_t *ksp;
 
    printf("\n=== kstat chain scan (zfs / unix / spl 関連) ===\n");
    printf("  %-20s %6s  %-35s  %s\n",
           "module", "inst", "name", "type");
    printf("  %-20s %6s  %-35s  %s\n",
           "--------------------", "------",
           "-----------------------------------", "----");
 
    for (ksp = kc->kc_chain; ksp != NULL; ksp = ksp->ks_next) {
        const char *m = ksp->ks_module;
        if (strcasecmp(m, "zfs")  == 0 ||
            strcasecmp(m, "unix") == 0 ||
            strcasecmp(m, "spl")  == 0 ||
            strcasecmp(m, "zvol") == 0) {
            printf("  %-20s %6d  %-35s  %d\n",
                   ksp->ks_module,
                   ksp->ks_instance,
                   ksp->ks_name,
                   ksp->ks_type);
        }
    }
}
 
/* -------------------------------------------------------
 * main
 * ------------------------------------------------------- */
int
main(void)
{
    kstat_ctl_t *kc;
 
    kc = kstat_open();
    if (kc == NULL) {
        fprintf(stderr, "kstat_open() failed: %s\n", strerror(errno));
        fprintf(stderr, "pfexec / root で実行してください。\n");
        return EXIT_FAILURE;
    }
 
    /* 1. チェーン全走査 */
    scan_chain(kc);
 
    /* 2. ZFS ARC 統計 (最重要) */
    dump_kstat(kc, "zfs", "arcstats");
 
    /* 3. ZFS I/O 統計候補 */
    dump_kstat(kc, "zfs", "zfetchstats");
    dump_kstat(kc, "zfs", "vdev_cache_stats");
    dump_kstat(kc, "zfs", "xuio_stats");
    dump_kstat(kc, "zfs", "fm");
    dump_kstat(kc, "zfs", "dmu_tx");
 
    /* 4. unix vmem (ZFS が使うメモリアリーナ) */
    dump_kstat(kc, "unix", "vmem");
 
    /* 5. spl (OpenZFS porting layer) */
    dump_kstat(kc, "spl",  "spl_misc");
 
    kstat_close(kc);
 
    printf("\n\n以上です。このログを共有してください。\n");
    return EXIT_SUCCESS;
}
 

コンパイルと実行方法は以下の取り。

sh

# コンパイル
cc -xc99 -o zfs_kstat_dump zfs_kstat_dump.c -lkstat

# 実行
pfexec ./zfs_kstat_dump 2>&1 | tee zfs_kstat_dump.log

ZFS kstat診断ツールの結果をプロンプトで貼り付け。


pfexec ./zfs_kstat_dump > zfs_kstat_dump.logの結果をお送りします。

<結果は省略させていただきます>


回答は以下のとおり。


ログファイルを読み込みます。
ZFS kstat診断ログを読む

ログを確認しました。使用可能なフィールドが明確になりました。本体を作成します。
ファイルを作成しました, コマンドを実行しました, ファイルを読み取りました
コンパイル・実行

sh

# コンパイル
cc  -xc99 -o zfs_monitor zfs_monitor.c -lkstat
# または
gcc -std=c99 -o zfs_monitor zfs_monitor.c -lkstat

# 実行
pfexec ./zfs_monitor            # 無限ループ
pfexec ./zfs_monitor 5000       # 5000サンプル取得して終了

出力の構成(1サンプル = 6行)
行 セクション 内容
1 [ARC Size/Usage] ARC現在量・上限・使用率、メタデータ使用率、プリフェッチバッファ、L2ARCサイズ(瞬時値)
2 [ARC Hit/Miss/ms] ARC全体・データ・メタデータ別ヒット/ミス数とヒット率(1ms差分)
3 [MRU/MFU Evict/ms] MRU/MFUリストのヒット数とエビクションバイト数(1ms差分)
4 [L2ARC /ms] L2ARCヒット/ミス数と読み書きバイト数(1ms差分)
5 [zfetch /ms] プリフェッチブロック数・ストリーム数・トリガーヒット数(1ms差分)
6 [vdev_cache /ms] vdev I/Oキャッシュのヒット/ミス数とヒット率(1ms差分)
kstat_dump.log で確認した実在フィールドのみ使用
kstat 使用フィールド
zfs/arcstats size, c_max, data_size, meta_used, meta_max, prefetch_size, hits, misses, demand_data_hits/misses, demand_metadata_hits/misses, mru_hits, mfu_hits, evicted_mfu/mru, l2_*
zfs/zfetchstats blocks_fetched, fetched_cached, new_streams, trigger_hits
zfs/vdev_cache_stats hits, misses

Zfs monitor
C

zfs_monitorのソースコードは以下のとおり。

/*
 * zfs_monitor.c
 *
 * Solaris 11.4 x86 向け
 * ZFS ARC (メモリキャッシュ) および I/O プリフェッチ統計を
 * 1 ミリ秒ごとに出力するプログラム。
 *
 * 使用する kstat (zfs_kstat_dump.log で実在確認済み):
 *
 *   [module=zfs, name=arcstats]  ks_ndata=76
 *     ARC サイズ系 (瞬時値):
 *       size          現在の ARC 使用量        (バイト)
 *       c             ARC ターゲットサイズ      (バイト)
 *       c_min         ARC 最小サイズ            (バイト)
 *       c_max         ARC 最大サイズ            (バイト)
 *       buf_size      ARC バッファヘッダ合計    (バイト)
 *       data_size     ARC データ領域            (バイト)
 *       other_size    ARC メタデータ等その他    (バイト)
 *       meta_used     ARC メタデータ使用量      (バイト)
 *       meta_max      ARC メタデータ最大使用量  (バイト)
 *       prefetch_size プリフェッチバッファ      (バイト)
 *       l2_size       L2ARC 現在サイズ          (バイト)
 *     ヒット / ミス (累積カウンタ):
 *       hits / misses
 *       demand_data_hits / demand_data_misses
 *       demand_metadata_hits / demand_metadata_misses
 *       mru_hits / mfu_hits
 *     エビクション (累積バイト):
 *       evicted_mfu / evicted_mru
 *     L2ARC (累積):
 *       l2_hits / l2_misses / l2_read_bytes / l2_write_bytes
 *
 *   [module=zfs, name=zfetchstats]  ks_ndata=11
 *       blocks_fetched / fetched_cached / new_streams / trigger_hits
 *
 *   [module=zfs, name=vdev_cache_stats]  ks_ndata=3
 *       hits / misses
 *
 * コンパイル:
 *   cc  -xc99 -o zfs_monitor zfs_monitor.c -lkstat
 *   gcc -std=c99 -o zfs_monitor zfs_monitor.c -lkstat
 *
 * 実行:
 *   pfexec ./zfs_monitor            # 無限ループ (Ctrl-C で停止)
 *   pfexec ./zfs_monitor 5000       # 5000 サンプル取得して終了
 */

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <errno.h>
#include <time.h>
#include <signal.h>
#include <kstat.h>
#include <sys/types.h>
#include <stdint.h>

#ifndef UINT64_MAX
#define UINT64_MAX ((uint64_t)0xFFFFFFFFFFFFFFFFULL)
#endif

/* ================================================================
 * 定数
 * ================================================================ */
#define INTERVAL_MS      1
#define NSEC_PER_MSEC    1000000L
#define HDR_EVERY        30   /* 何サンプルごとにヘッダーを再表示するか */

/* ================================================================
 * kstat スナップショット構造体
 * ================================================================ */

typedef struct arc_snap {
    /* 瞬時値 */
    uint64_t size;
    uint64_t c;
    uint64_t c_min;
    uint64_t c_max;
    uint64_t buf_size;
    uint64_t data_size;
    uint64_t other_size;
    uint64_t meta_used;
    uint64_t meta_max;
    uint64_t prefetch_size;
    uint64_t l2_size;
    /* 累積カウンタ */
    uint64_t hits;
    uint64_t misses;
    uint64_t demand_data_hits;
    uint64_t demand_data_misses;
    uint64_t demand_meta_hits;
    uint64_t demand_meta_misses;
    uint64_t mru_hits;
    uint64_t mfu_hits;
    uint64_t evicted_mfu;
    uint64_t evicted_mru;
    uint64_t l2_hits;
    uint64_t l2_misses;
    uint64_t l2_read_bytes;
    uint64_t l2_write_bytes;
} arc_snap_t;

typedef struct zfetch_snap {
    uint64_t blocks_fetched;
    uint64_t fetched_cached;
    uint64_t new_streams;
    uint64_t trigger_hits;
} zfetch_snap_t;

typedef struct vcache_snap {
    uint64_t hits;
    uint64_t misses;
} vcache_snap_t;

/* ================================================================
 * 表示用統計
 * ================================================================ */
typedef struct arc_stat {
    /* サイズ系 (瞬時値) */
    uint64_t size;
    uint64_t c_max;
    uint64_t data_size;
    uint64_t meta_used;
    uint64_t meta_max;
    uint64_t prefetch_size;
    uint64_t l2_size;
    double   arc_pct;
    double   meta_pct;
    /* ARC ヒット/ミス (1ms 差分) */
    uint64_t d_hits;
    uint64_t d_misses;
    uint64_t d_demand_data_hits;
    uint64_t d_demand_data_misses;
    uint64_t d_demand_meta_hits;
    uint64_t d_demand_meta_misses;
    uint64_t d_mru_hits;
    uint64_t d_mfu_hits;
    uint64_t d_evicted_mfu;
    uint64_t d_evicted_mru;
    uint64_t d_l2_hits;
    uint64_t d_l2_misses;
    uint64_t d_l2_read_bytes;
    uint64_t d_l2_write_bytes;
    double   hit_rate;
    double   data_hit_rate;
    double   meta_hit_rate;
    /* zfetch (1ms 差分) */
    uint64_t d_blocks_fetched;
    uint64_t d_fetched_cached;
    uint64_t d_new_streams;
    uint64_t d_trigger_hits;
    /* vdev_cache (1ms 差分) */
    uint64_t d_vc_hits;
    uint64_t d_vc_misses;
    double   vc_hit_rate;
} arc_stat_t;

/* ================================================================
 * グローバル / シグナル
 * ================================================================ */
static volatile sig_atomic_t g_running = 1;
static void sig_handler(int s) { (void)s; g_running = 0; }

/* ================================================================
 * kstat ユーティリティ
 * ================================================================ */
static int
ks_get_u64(kstat_t *ksp, const char *field, uint64_t *out)
{
    kstat_named_t *knp =
        (kstat_named_t *)kstat_data_lookup(ksp, (char *)field);
    if (!knp) return -1;
    switch (knp->data_type) {
    case KSTAT_DATA_UINT64: *out = knp->value.ui64;            break;
    case KSTAT_DATA_INT64:  *out = (uint64_t)knp->value.i64;  break;
    case KSTAT_DATA_UINT32: *out = (uint64_t)knp->value.ui32; break;
    case KSTAT_DATA_INT32:  *out = (uint64_t)knp->value.i32;  break;
    default: return -1;
    }
    return 0;
}

/* ================================================================
 * kstat 読み取り
 * ================================================================ */
static int
read_arcstats(kstat_ctl_t *kc, arc_snap_t *s)
{
    kstat_t *ksp = kstat_lookup(kc, "zfs", -1, "arcstats");
    if (!ksp) {
        fprintf(stderr, "[ERROR] kstat_lookup(zfs/arcstats): %s\n",
                strerror(errno));
        return -1;
    }
    if (kstat_read(kc, ksp, NULL) == -1) {
        fprintf(stderr, "[ERROR] kstat_read(arcstats): %s\n",
                strerror(errno));
        return -1;
    }
    memset(s, 0, sizeof(*s));
    ks_get_u64(ksp, "size",                   &s->size);
    ks_get_u64(ksp, "c",                      &s->c);
    ks_get_u64(ksp, "c_min",                  &s->c_min);
    ks_get_u64(ksp, "c_max",                  &s->c_max);
    ks_get_u64(ksp, "buf_size",               &s->buf_size);
    ks_get_u64(ksp, "data_size",              &s->data_size);
    ks_get_u64(ksp, "other_size",             &s->other_size);
    ks_get_u64(ksp, "meta_used",              &s->meta_used);
    ks_get_u64(ksp, "meta_max",               &s->meta_max);
    ks_get_u64(ksp, "prefetch_size",          &s->prefetch_size);
    ks_get_u64(ksp, "l2_size",                &s->l2_size);
    ks_get_u64(ksp, "hits",                   &s->hits);
    ks_get_u64(ksp, "misses",                 &s->misses);
    ks_get_u64(ksp, "demand_data_hits",       &s->demand_data_hits);
    ks_get_u64(ksp, "demand_data_misses",     &s->demand_data_misses);
    ks_get_u64(ksp, "demand_metadata_hits",   &s->demand_meta_hits);
    ks_get_u64(ksp, "demand_metadata_misses", &s->demand_meta_misses);
    ks_get_u64(ksp, "mru_hits",               &s->mru_hits);
    ks_get_u64(ksp, "mfu_hits",               &s->mfu_hits);
    ks_get_u64(ksp, "evicted_mfu",            &s->evicted_mfu);
    ks_get_u64(ksp, "evicted_mru",            &s->evicted_mru);
    ks_get_u64(ksp, "l2_hits",                &s->l2_hits);
    ks_get_u64(ksp, "l2_misses",              &s->l2_misses);
    ks_get_u64(ksp, "l2_read_bytes",          &s->l2_read_bytes);
    ks_get_u64(ksp, "l2_write_bytes",         &s->l2_write_bytes);
    return 0;
}

static int
read_zfetch(kstat_ctl_t *kc, zfetch_snap_t *s)
{
    kstat_t *ksp = kstat_lookup(kc, "zfs", -1, "zfetchstats");
    if (!ksp || kstat_read(kc, ksp, NULL) == -1) return -1;
    memset(s, 0, sizeof(*s));
    ks_get_u64(ksp, "blocks_fetched", &s->blocks_fetched);
    ks_get_u64(ksp, "fetched_cached", &s->fetched_cached);
    ks_get_u64(ksp, "new_streams",    &s->new_streams);
    ks_get_u64(ksp, "trigger_hits",   &s->trigger_hits);
    return 0;
}

static int
read_vcache(kstat_ctl_t *kc, vcache_snap_t *s)
{
    kstat_t *ksp = kstat_lookup(kc, "zfs", -1, "vdev_cache_stats");
    if (!ksp || kstat_read(kc, ksp, NULL) == -1) return -1;
    memset(s, 0, sizeof(*s));
    ks_get_u64(ksp, "hits",   &s->hits);
    ks_get_u64(ksp, "misses", &s->misses);
    return 0;
}

/* ================================================================
 * 差分 (カウンタ折り返し対策)
 * ================================================================ */
static uint64_t
delta64(uint64_t prev, uint64_t curr)
{
    return (curr >= prev) ? (curr - prev)
                          : (UINT64_MAX - prev + curr + 1);
}

/* ================================================================
 * 統計計算
 * ================================================================ */
static void
calc_stat(const arc_snap_t    *ap, const arc_snap_t    *ac,
          const zfetch_snap_t *zp, const zfetch_snap_t *zc,
          const vcache_snap_t *vp, const vcache_snap_t *vc,
          arc_stat_t *o)
{
    uint64_t total;
    memset(o, 0, sizeof(*o));

    /* 瞬時値 */
    o->size          = ac->size;
    o->c_max         = ac->c_max;
    o->data_size     = ac->data_size;
    o->meta_used     = ac->meta_used;
    o->meta_max      = ac->meta_max;
    o->prefetch_size = ac->prefetch_size;
    o->l2_size       = ac->l2_size;

    o->arc_pct  = (ac->c_max > 0)
        ? (double)ac->size     / (double)ac->c_max   * 100.0 : 0.0;
    o->meta_pct = (ac->meta_max > 0)
        ? (double)ac->meta_used / (double)ac->meta_max * 100.0 : 0.0;

    /* 1ms 差分 */
    o->d_hits               = delta64(ap->hits,               ac->hits);
    o->d_misses             = delta64(ap->misses,             ac->misses);
    o->d_demand_data_hits   = delta64(ap->demand_data_hits,   ac->demand_data_hits);
    o->d_demand_data_misses = delta64(ap->demand_data_misses, ac->demand_data_misses);
    o->d_demand_meta_hits   = delta64(ap->demand_meta_hits,   ac->demand_meta_hits);
    o->d_demand_meta_misses = delta64(ap->demand_meta_misses, ac->demand_meta_misses);
    o->d_mru_hits           = delta64(ap->mru_hits,           ac->mru_hits);
    o->d_mfu_hits           = delta64(ap->mfu_hits,           ac->mfu_hits);
    o->d_evicted_mfu        = delta64(ap->evicted_mfu,        ac->evicted_mfu);
    o->d_evicted_mru        = delta64(ap->evicted_mru,        ac->evicted_mru);
    o->d_l2_hits            = delta64(ap->l2_hits,            ac->l2_hits);
    o->d_l2_misses          = delta64(ap->l2_misses,          ac->l2_misses);
    o->d_l2_read_bytes      = delta64(ap->l2_read_bytes,      ac->l2_read_bytes);
    o->d_l2_write_bytes     = delta64(ap->l2_write_bytes,     ac->l2_write_bytes);

    total = o->d_hits + o->d_misses;
    o->hit_rate = (total > 0)
        ? (double)o->d_hits / (double)total * 100.0 : -1.0;

    total = o->d_demand_data_hits + o->d_demand_data_misses;
    o->data_hit_rate = (total > 0)
        ? (double)o->d_demand_data_hits / (double)total * 100.0 : -1.0;

    total = o->d_demand_meta_hits + o->d_demand_meta_misses;
    o->meta_hit_rate = (total > 0)
        ? (double)o->d_demand_meta_hits / (double)total * 100.0 : -1.0;

    /* zfetch */
    o->d_blocks_fetched = delta64(zp->blocks_fetched, zc->blocks_fetched);
    o->d_fetched_cached = delta64(zp->fetched_cached, zc->fetched_cached);
    o->d_new_streams    = delta64(zp->new_streams,    zc->new_streams);
    o->d_trigger_hits   = delta64(zp->trigger_hits,   zc->trigger_hits);

    /* vdev_cache */
    o->d_vc_hits   = delta64(vp->hits,   vc->hits);
    o->d_vc_misses = delta64(vp->misses, vc->misses);
    total = o->d_vc_hits + o->d_vc_misses;
    o->vc_hit_rate = (total > 0)
        ? (double)o->d_vc_hits / (double)total * 100.0 : -1.0;
}

/* ================================================================
 * ユーティリティ
 * ================================================================ */
static void
timestamp(char *buf, size_t n)
{
    struct timespec ts;
    struct tm       tm;
    clock_gettime(CLOCK_REALTIME, &ts);
    localtime_r(&ts.tv_sec, &tm);
    snprintf(buf, n, "%02d:%02d:%02d.%09ld",
             tm.tm_hour, tm.tm_min, tm.tm_sec, ts.tv_nsec);
}

static void
sleep_1ms(void)
{
    struct timespec req = { 0, NSEC_PER_MSEC };
    while (clock_nanosleep(CLOCK_MONOTONIC, 0, &req, &req) == EINTR)
        if (!g_running) break;
}

/* バイトを K/M/G 単位で 7 文字以内に整形 */
static void
fmt_bytes(char *buf, size_t n, uint64_t v)
{
    if (v >= (uint64_t)1024*1024*1024)
        snprintf(buf, n, "%6.2fG", (double)v / (1024.0*1024*1024));
    else if (v >= 1024*1024)
        snprintf(buf, n, "%6.2fM", (double)v / (1024.0*1024));
    else if (v >= 1024)
        snprintf(buf, n, "%6.2fK", (double)v / 1024.0);
    else
        snprintf(buf, n, "%6lluB",  (unsigned long long)v);
}

/* ヒット率 (-1 なら "    N/A") */
static void
fmt_rate(char *buf, size_t n, double r)
{
    if (r < 0.0) snprintf(buf, n, "    N/A");
    else         snprintf(buf, n, "%6.2f%%", r);
}

/* ================================================================
 * 表示
 * ================================================================ */
#define SEP "  "

static void
print_header(void)
{
    puts("");
    /* ヘッダー行 */
    printf("%-19s" SEP
           "%-9s %-9s %-8s" SEP
           "%-9s %-9s %-8s" SEP
           "%-9s %-9s\n",
           "[ARC Size/Usage]",
           "ARC_used","ARC_max","ARC(%)",
           "Meta_used","Meta_max","Meta(%)",
           "Pfetch","L2ARC");

    printf("%-19s" SEP
           "%-7s %-7s %-8s" SEP
           "%-7s %-7s %-8s" SEP
           "%-7s %-7s %-8s\n",
           "[ARC Hit/Miss/ms]",
           "Hits","Misses","HitRate",
           "D_Hits","D_Miss","D_Rate",
           "M_Hits","M_Miss","M_Rate");

    printf("%-19s" SEP
           "%-7s %-7s" SEP
           "%-9s %-9s\n",
           "[MRU/MFU Evict/ms]",
           "MRU_hit","MFU_hit",
           "MRU_evict","MFU_evict");

    printf("%-19s" SEP
           "%-7s %-7s" SEP
           "%-9s %-9s\n",
           "[L2ARC /ms]",
           "L2_hits","L2_miss",
           "L2_rdB","L2_wrB");

    printf("%-19s" SEP
           "%-8s %-8s %-8s %-8s\n",
           "[zfetch /ms]",
           "BlkFetch","Cached","NewStrm","TrigHit");

    printf("%-19s" SEP
           "%-7s %-7s %-8s\n",
           "[vdev_cache /ms]",
           "VC_hits","VC_miss","VC_Rate");

    puts("------------------------------------------------------------"
         "------------------------------------------------------------");
}

static void
print_stat(const char *ts, const arc_stat_t *s)
{
    char b1[16],b2[16],b3[16],b4[16],b5[16],b6[16];
    char r1[16],r2[16],r3[16],r4[16];

    /* ARC Size/Usage */
    fmt_bytes(b1,sizeof(b1),s->size);
    fmt_bytes(b2,sizeof(b2),s->c_max);
    fmt_rate (r1,sizeof(r1),s->arc_pct);
    fmt_bytes(b3,sizeof(b3),s->meta_used);
    fmt_bytes(b4,sizeof(b4),s->meta_max);
    fmt_rate (r2,sizeof(r2),s->meta_pct);
    fmt_bytes(b5,sizeof(b5),s->prefetch_size);
    fmt_bytes(b6,sizeof(b6),s->l2_size);
    printf("%-19s" SEP "%s %s %s" SEP "%s %s %s" SEP "%s %s\n",
           ts, b1,b2,r1, b3,b4,r2, b5,b6);

    /* ARC Hit/Miss */
    fmt_rate(r1,sizeof(r1),s->hit_rate);
    fmt_rate(r2,sizeof(r2),s->data_hit_rate);
    fmt_rate(r3,sizeof(r3),s->meta_hit_rate);
    printf("%-19s" SEP
           "%7llu %7llu %s" SEP
           "%7llu %7llu %s" SEP
           "%7llu %7llu %s\n",
           ts,
           (unsigned long long)s->d_hits,
           (unsigned long long)s->d_misses,   r1,
           (unsigned long long)s->d_demand_data_hits,
           (unsigned long long)s->d_demand_data_misses, r2,
           (unsigned long long)s->d_demand_meta_hits,
           (unsigned long long)s->d_demand_meta_misses, r3);

    /* MRU/MFU Evict */
    fmt_bytes(b1,sizeof(b1),s->d_evicted_mru);
    fmt_bytes(b2,sizeof(b2),s->d_evicted_mfu);
    printf("%-19s" SEP "%7llu %7llu" SEP "%s %s\n",
           ts,
           (unsigned long long)s->d_mru_hits,
           (unsigned long long)s->d_mfu_hits,
           b1, b2);

    /* L2ARC */
    fmt_bytes(b1,sizeof(b1),s->d_l2_read_bytes);
    fmt_bytes(b2,sizeof(b2),s->d_l2_write_bytes);
    printf("%-19s" SEP "%7llu %7llu" SEP "%s %s\n",
           ts,
           (unsigned long long)s->d_l2_hits,
           (unsigned long long)s->d_l2_misses,
           b1, b2);

    /* zfetch */
    printf("%-19s" SEP "%8llu %8llu %8llu %8llu\n",
           ts,
           (unsigned long long)s->d_blocks_fetched,
           (unsigned long long)s->d_fetched_cached,
           (unsigned long long)s->d_new_streams,
           (unsigned long long)s->d_trigger_hits);

    /* vdev_cache */
    fmt_rate(r4,sizeof(r4),s->vc_hit_rate);
    printf("%-19s" SEP "%7llu %7llu %s\n\n",
           ts,
           (unsigned long long)s->d_vc_hits,
           (unsigned long long)s->d_vc_misses,
           r4);
}

/* ================================================================
 * 凡例
 * ================================================================ */
static void
print_legend(void)
{
    puts(
"凡例:\n"
"  [ARC Size/Usage]  ← 瞬時値\n"
"    ARC_used   : ARC 現在使用量 (size)\n"
"    ARC_max    : ARC 最大サイズ (c_max)\n"
"    ARC(%%)     : ARC 使用率 = size / c_max\n"
"    Meta_used  : ARC メタデータ現在使用量\n"
"    Meta_max   : ARC メタデータ最大使用量\n"
"    Meta(%%)    : メタデータ使用率\n"
"    Pfetch     : プリフェッチバッファ現在サイズ\n"
"    L2ARC      : L2ARC (2次ディスクキャッシュ) 現在サイズ\n"
"\n"
"  [ARC Hit/Miss/ms]  ← 1ms 差分\n"
"    Hits/Misses : ARC 全体ヒット/ミス数\n"
"    HitRate     : ARC 全体ヒット率\n"
"    D_Hits/Miss : demand_data ヒット/ミス数\n"
"    D_Rate      : データヒット率\n"
"    M_Hits/Miss : demand_metadata ヒット/ミス数\n"
"    M_Rate      : メタデータヒット率\n"
"    N/A         : 1ms 間でアクセスなし\n"
"\n"
"  [MRU/MFU Evict/ms]  ← 1ms 差分\n"
"    MRU_hit    : MRU リストヒット数\n"
"    MFU_hit    : MFU リストヒット数\n"
"    MRU_evict  : MRU からエビクトされたバイト数\n"
"    MFU_evict  : MFU からエビクトされたバイト数\n"
"\n"
"  [L2ARC /ms]  ← 1ms 差分\n"
"    L2_hits/miss : L2ARC ヒット/ミス数\n"
"    L2_rdB/wrB   : L2ARC 読み書きバイト数\n"
"\n"
"  [zfetch /ms]  ← 1ms 差分\n"
"    BlkFetch : プリフェッチ取得ブロック数\n"
"    Cached   : キャッシュ済みプリフェッチ数\n"
"    NewStrm  : 新規プリフェッチストリーム数\n"
"    TrigHit  : プリフェッチトリガーヒット数\n"
"\n"
"  [vdev_cache /ms]  ← 1ms 差分\n"
"    VC_hits/miss : vdev I/O キャッシュ ヒット/ミス数\n"
"    VC_Rate      : vdev キャッシュヒット率\n"
    );
}

/* ================================================================
 * main
 * ================================================================ */
int
main(int argc, char *argv[])
{
    kstat_ctl_t   *kc;
    arc_snap_t     arc_prev, arc_curr;
    zfetch_snap_t  zf_prev,  zf_curr;
    vcache_snap_t  vc_prev,  vc_curr;
    arc_stat_t     stat;
    char           ts[32];
    long           max_iter = -1;
    long           iter     = 0;
    int            hdr_cnt  = 0;

    if (argc >= 2) {
        max_iter = atol(argv[1]);
        if (max_iter <= 0) {
            fprintf(stderr, "Usage: %s [count]\n", argv[0]);
            return EXIT_FAILURE;
        }
    }

    signal(SIGINT,  sig_handler);
    signal(SIGTERM, sig_handler);

    kc = kstat_open();
    if (!kc) {
        fprintf(stderr, "kstat_open() failed: %s\n", strerror(errno));
        fprintf(stderr, "pfexec / root で実行してください。\n");
        return EXIT_FAILURE;
    }

    printf("Solaris 11.4 ZFS Monitor  [interval=%dms]\n", INTERVAL_MS);
    printf("Ctrl-C で停止\n\n");
    print_legend();

    /* ベースライン取得 */
    if (read_arcstats(kc, &arc_prev) != 0 ||
        read_zfetch(kc,   &zf_prev)  != 0 ||
        read_vcache(kc,   &vc_prev)  != 0) {
        kstat_close(kc);
        return EXIT_FAILURE;
    }

    sleep_1ms();

    /* メインループ */
    while (g_running) {

        if (kstat_chain_update(kc) == -1)
            fprintf(stderr, "[WARN] kstat_chain_update: %s\n",
                    strerror(errno));

        if (read_arcstats(kc, &arc_curr) != 0 ||
            read_zfetch(kc,   &zf_curr)  != 0 ||
            read_vcache(kc,   &vc_curr)  != 0)
            break;

        calc_stat(&arc_prev, &arc_curr,
                  &zf_prev,  &zf_curr,
                  &vc_prev,  &vc_curr,
                  &stat);

        timestamp(ts, sizeof(ts));

        if (hdr_cnt % HDR_EVERY == 0)
            print_header();
        hdr_cnt++;

        print_stat(ts, &stat);

        arc_prev = arc_curr;
        zf_prev  = zf_curr;
        vc_prev  = vc_curr;

        iter++;
        if (max_iter > 0 && iter >= max_iter)
            break;

        sleep_1ms();
    }

    kstat_close(kc);
    printf("合計 %ld サンプル取得\n", iter);
    return EXIT_SUCCESS;
}