-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathnvme.h
467 lines (414 loc) · 8.43 KB
/
nvme.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
// SPDX-License-Identifier: GPL-2.0
/*
* nvme structure declarations and helper functions for the
* io_uring_cmd engine.
*/
#ifndef FIO_NVME_H
#define FIO_NVME_H
#include <linux/nvme_ioctl.h>
#include "../fio.h"
/*
* If the uapi headers installed on the system lacks nvme uring command
* support, use the local version to prevent compilation issues.
*/
#ifndef CONFIG_NVME_URING_CMD
struct nvme_uring_cmd {
__u8 opcode;
__u8 flags;
__u16 rsvd1;
__u32 nsid;
__u32 cdw2;
__u32 cdw3;
__u64 metadata;
__u64 addr;
__u32 metadata_len;
__u32 data_len;
__u32 cdw10;
__u32 cdw11;
__u32 cdw12;
__u32 cdw13;
__u32 cdw14;
__u32 cdw15;
__u32 timeout_ms;
__u32 rsvd2;
};
#define NVME_URING_CMD_IO _IOWR('N', 0x80, struct nvme_uring_cmd)
#define NVME_URING_CMD_IO_VEC _IOWR('N', 0x81, struct nvme_uring_cmd)
#endif /* CONFIG_NVME_URING_CMD */
#define NVME_DEFAULT_IOCTL_TIMEOUT 0
#define NVME_IDENTIFY_DATA_SIZE 4096
#define NVME_IDENTIFY_CSI_SHIFT 24
#define NVME_NQN_LENGTH 256
#define NVME_PI_APP_DISABLE 0xFFFF
#define NVME_PI_REF_DISABLE 0xFFFFFFFF
#define NVME_ZNS_ZRA_REPORT_ZONES 0
#define NVME_ZNS_ZRAS_FEAT_ERZ (1 << 16)
#define NVME_ZNS_ZSA_RESET 0x4
#define NVME_ZONE_TYPE_SEQWRITE_REQ 0x2
#define NVME_ATTRIBUTE_DEALLOCATE (1 << 2)
enum nvme_identify_cns {
NVME_IDENTIFY_CNS_NS = 0x00,
NVME_IDENTIFY_CNS_CTRL = 0x01,
NVME_IDENTIFY_CNS_CSI_NS = 0x05,
NVME_IDENTIFY_CNS_CSI_CTRL = 0x06,
};
enum nvme_csi {
NVME_CSI_NVM = 0,
NVME_CSI_KV = 1,
NVME_CSI_ZNS = 2,
};
enum nvme_admin_opcode {
nvme_admin_identify = 0x06,
};
enum nvme_io_opcode {
nvme_cmd_write = 0x01,
nvme_cmd_read = 0x02,
nvme_cmd_dsm = 0x09,
nvme_cmd_io_mgmt_recv = 0x12,
nvme_zns_cmd_mgmt_send = 0x79,
nvme_zns_cmd_mgmt_recv = 0x7a,
};
enum nvme_zns_zs {
NVME_ZNS_ZS_EMPTY = 0x1,
NVME_ZNS_ZS_IMPL_OPEN = 0x2,
NVME_ZNS_ZS_EXPL_OPEN = 0x3,
NVME_ZNS_ZS_CLOSED = 0x4,
NVME_ZNS_ZS_READ_ONLY = 0xd,
NVME_ZNS_ZS_FULL = 0xe,
NVME_ZNS_ZS_OFFLINE = 0xf,
};
enum nvme_id_ctrl_ctratt {
NVME_CTRL_CTRATT_ELBAS = 1 << 15,
};
enum {
NVME_ID_NS_NVM_STS_MASK = 0x7f,
NVME_ID_NS_NVM_GUARD_SHIFT = 7,
NVME_ID_NS_NVM_GUARD_MASK = 0x3,
};
enum {
NVME_NVM_NS_16B_GUARD = 0,
NVME_NVM_NS_32B_GUARD = 1,
NVME_NVM_NS_64B_GUARD = 2,
};
struct nvme_data {
__u32 nsid;
__u32 lba_shift;
__u32 lba_size;
__u32 lba_ext;
__u16 ms;
__u16 pi_size;
__u8 pi_type;
__u8 guard_type;
__u8 pi_loc;
};
enum nvme_id_ns_dps {
NVME_NS_DPS_PI_NONE = 0,
NVME_NS_DPS_PI_TYPE1 = 1,
NVME_NS_DPS_PI_TYPE2 = 2,
NVME_NS_DPS_PI_TYPE3 = 3,
NVME_NS_DPS_PI_MASK = 7 << 0,
NVME_NS_DPS_PI_FIRST = 1 << 3,
};
enum nvme_io_control_flags {
NVME_IO_PRINFO_PRCHK_REF = 1U << 26,
NVME_IO_PRINFO_PRCHK_APP = 1U << 27,
NVME_IO_PRINFO_PRCHK_GUARD = 1U << 28,
NVME_IO_PRINFO_PRACT = 1U << 29,
};
struct nvme_pi_data {
__u32 interval;
__u32 io_flags;
__u16 apptag;
__u16 apptag_mask;
};
struct nvme_lbaf {
__le16 ms;
__u8 ds;
__u8 rp;
};
/* 16 bit guard protection Information format */
struct nvme_16b_guard_pif {
__be16 guard;
__be16 apptag;
__be32 srtag;
};
/* 64 bit guard protection Information format */
struct nvme_64b_guard_pif {
__be64 guard;
__be16 apptag;
__u8 srtag[6];
};
struct nvme_id_ns {
__le64 nsze;
__le64 ncap;
__le64 nuse;
__u8 nsfeat;
__u8 nlbaf;
__u8 flbas;
__u8 mc;
__u8 dpc;
__u8 dps;
__u8 nmic;
__u8 rescap;
__u8 fpi;
__u8 dlfeat;
__le16 nawun;
__le16 nawupf;
__le16 nacwu;
__le16 nabsn;
__le16 nabo;
__le16 nabspf;
__le16 noiob;
__u8 nvmcap[16];
__le16 npwg;
__le16 npwa;
__le16 npdg;
__le16 npda;
__le16 nows;
__le16 mssrl;
__le32 mcl;
__u8 msrc;
__u8 rsvd81[11];
__le32 anagrpid;
__u8 rsvd96[3];
__u8 nsattr;
__le16 nvmsetid;
__le16 endgid;
__u8 nguid[16];
__u8 eui64[8];
struct nvme_lbaf lbaf[64];
__u8 vs[3712];
};
struct nvme_id_psd {
__le16 mp;
__u8 rsvd2;
__u8 flags;
__le32 enlat;
__le32 exlat;
__u8 rrt;
__u8 rrl;
__u8 rwt;
__u8 rwl;
__le16 idlp;
__u8 ips;
__u8 rsvd19;
__le16 actp;
__u8 apws;
__u8 rsvd23[9];
};
struct nvme_id_ctrl {
__le16 vid;
__le16 ssvid;
char sn[20];
char mn[40];
char fr[8];
__u8 rab;
__u8 ieee[3];
__u8 cmic;
__u8 mdts;
__le16 cntlid;
__le32 ver;
__le32 rtd3r;
__le32 rtd3e;
__le32 oaes;
__le32 ctratt;
__le16 rrls;
__u8 rsvd102[9];
__u8 cntrltype;
__u8 fguid[16];
__le16 crdt1;
__le16 crdt2;
__le16 crdt3;
__u8 rsvd134[119];
__u8 nvmsr;
__u8 vwci;
__u8 mec;
__le16 oacs;
__u8 acl;
__u8 aerl;
__u8 frmw;
__u8 lpa;
__u8 elpe;
__u8 npss;
__u8 avscc;
__u8 apsta;
__le16 wctemp;
__le16 cctemp;
__le16 mtfa;
__le32 hmpre;
__le32 hmmin;
__u8 tnvmcap[16];
__u8 unvmcap[16];
__le32 rpmbs;
__le16 edstt;
__u8 dsto;
__u8 fwug;
__le16 kas;
__le16 hctma;
__le16 mntmt;
__le16 mxtmt;
__le32 sanicap;
__le32 hmminds;
__le16 hmmaxd;
__le16 nsetidmax;
__le16 endgidmax;
__u8 anatt;
__u8 anacap;
__le32 anagrpmax;
__le32 nanagrpid;
__le32 pels;
__le16 domainid;
__u8 rsvd358[10];
__u8 megcap[16];
__u8 rsvd384[128];
__u8 sqes;
__u8 cqes;
__le16 maxcmd;
__le32 nn;
__le16 oncs;
__le16 fuses;
__u8 fna;
__u8 vwc;
__le16 awun;
__le16 awupf;
__u8 icsvscc;
__u8 nwpc;
__le16 acwu;
__le16 ocfs;
__le32 sgls;
__le32 mnan;
__u8 maxdna[16];
__le32 maxcna;
__u8 rsvd564[204];
char subnqn[NVME_NQN_LENGTH];
__u8 rsvd1024[768];
/* Fabrics Only */
__le32 ioccsz;
__le32 iorcsz;
__le16 icdoff;
__u8 fcatt;
__u8 msdbd;
__le16 ofcs;
__u8 dctype;
__u8 rsvd1807[241];
struct nvme_id_psd psd[32];
__u8 vs[1024];
};
struct nvme_nvm_id_ns {
__le64 lbstm;
__u8 pic;
__u8 rsvd9[3];
__le32 elbaf[64];
__u8 rsvd268[3828];
};
static inline int ilog2(uint32_t i)
{
int log = -1;
while (i) {
i >>= 1;
log++;
}
return log;
}
struct nvme_zns_lbafe {
__le64 zsze;
__u8 zdes;
__u8 rsvd9[7];
};
struct nvme_zns_id_ns {
__le16 zoc;
__le16 ozcs;
__le32 mar;
__le32 mor;
__le32 rrl;
__le32 frl;
__le32 rrl1;
__le32 rrl2;
__le32 rrl3;
__le32 frl1;
__le32 frl2;
__le32 frl3;
__le32 numzrwa;
__le16 zrwafg;
__le16 zrwasz;
__u8 zrwacap;
__u8 rsvd53[2763];
struct nvme_zns_lbafe lbafe[64];
__u8 vs[256];
};
struct nvme_zns_desc {
__u8 zt;
__u8 zs;
__u8 za;
__u8 zai;
__u8 rsvd4[4];
__le64 zcap;
__le64 zslba;
__le64 wp;
__u8 rsvd32[32];
};
struct nvme_zone_report {
__le64 nr_zones;
__u8 rsvd8[56];
struct nvme_zns_desc entries[];
};
struct nvme_fdp_ruh_status_desc {
__u16 pid;
__u16 ruhid;
__u32 earutr;
__u64 ruamw;
__u8 rsvd16[16];
};
struct nvme_fdp_ruh_status {
__u8 rsvd0[14];
__le16 nruhsd;
struct nvme_fdp_ruh_status_desc ruhss[];
};
struct nvme_dsm_range {
__le32 cattr;
__le32 nlb;
__le64 slba;
};
struct nvme_cmd_ext_io_opts {
__u32 io_flags;
__u16 apptag;
__u16 apptag_mask;
};
int fio_nvme_iomgmt_ruhs(struct thread_data *td, struct fio_file *f,
struct nvme_fdp_ruh_status *ruhs, __u32 bytes);
int fio_nvme_get_info(struct fio_file *f, __u64 *nlba, __u32 pi_act,
struct nvme_data *data);
int fio_nvme_uring_cmd_prep(struct nvme_uring_cmd *cmd, struct io_u *io_u,
struct iovec *iov, struct nvme_dsm_range *dsm);
void fio_nvme_pi_fill(struct nvme_uring_cmd *cmd, struct io_u *io_u,
struct nvme_cmd_ext_io_opts *opts);
int fio_nvme_pi_verify(struct nvme_data *data, struct io_u *io_u);
int fio_nvme_get_zoned_model(struct thread_data *td, struct fio_file *f,
enum zbd_zoned_model *model);
int fio_nvme_report_zones(struct thread_data *td, struct fio_file *f,
uint64_t offset, struct zbd_zone *zbdz,
unsigned int nr_zones);
int fio_nvme_reset_wp(struct thread_data *td, struct fio_file *f,
uint64_t offset, uint64_t length);
int fio_nvme_get_max_open_zones(struct thread_data *td, struct fio_file *f,
unsigned int *max_open_zones);
static inline void put_unaligned_be48(__u64 val, __u8 *p)
{
*p++ = val >> 40;
*p++ = val >> 32;
*p++ = val >> 24;
*p++ = val >> 16;
*p++ = val >> 8;
*p++ = val;
}
static inline __u64 get_unaligned_be48(__u8 *p)
{
return (__u64)p[0] << 40 | (__u64)p[1] << 32 | (__u64)p[2] << 24 |
p[3] << 16 | p[4] << 8 | p[5];
}
static inline bool fio_nvme_pi_ref_escape(__u8 *reftag)
{
__u8 ref_esc[6] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff};
return memcmp(reftag, ref_esc, sizeof(ref_esc)) == 0;
}
#endif