Re: [PATCH] Allow verification to be done at finer granularity

From: Shawn Lewis <shawnlewis_at_google.com>
Date: Fri, 27 Jul 2007 13:28:21 -0700

Whoops, I keep generating patches before running git-update-index again.

Please remove the last line in populate_verify_io_u:

"+ memcpy(p, hdr, sizeof(hdr));"

On 7/27/07, Shawn Lewis <shawnlewis_at_google.com> wrote:
>
> Allow verification to be done at a finer granularity than blocksize.
>
> diff --git a/HOWTO b/HOWTO
> index b862d74..2797a58 100644
> --- a/HOWTO
> +++ b/HOWTO
> @@ -602,6 +602,11 @@ verifysort=bool If set, fio will sort wr
> can ignore this option unless doing huge amounts of really
> fast IO where the red-black tree sorting CPU time becomes
> significant.
> +
> +header_interval=siint Write the verification header at a finer
> granularity
> + than the blocksize. It will be written for chunks
> the
> + size of header_interval. blocksize should divide
> this
> + evenly.
>
> stonewall Wait for preceeding jobs in the job file to exit, before
> starting this one. Can be used to insert serialization
> diff --git a/fio.h b/fio.h
> index 1173bde..653f502 100644
> --- a/fio.h
> +++ b/fio.h
> @@ -395,6 +395,7 @@ struct thread_options {
> unsigned int sync_io;
> unsigned int verify;
> unsigned int verifysort;
> + unsigned int header_interval;
> unsigned int use_thread;
> unsigned int unlink;
> unsigned int do_disk_util;
> diff --git a/options.c b/options.c
> index 3bf0d24..829b808 100644
> --- a/options.c
> +++ b/options.c
> @@ -604,6 +604,13 @@ #endif
> .def = "1",
> },
> {
> + .name = "header_interval",
> + .type = FIO_OPT_STR_VAL_INT,
> + .off1 = td_var_offset(header_interval),
> + .help = "Store buffer header every N bytes",
> + .def = "0",
> + },
> + {
> .name = "write_iolog",
> .type = FIO_OPT_STR_STORE,
> .off1 = td_var_offset(write_iolog_file),
> diff --git a/verify.c b/verify.c
> index 156f068..daa7256 100644
> --- a/verify.c
> +++ b/verify.c
> @@ -42,16 +42,19 @@ static void hexdump(void *buffer, int le
> log_info("\n");
> }
>
> -static int verify_io_u_crc7(struct verify_header *hdr, struct io_u *io_u)
> +static int verify_io_u_crc7(struct verify_header *hdr, struct io_u *io_u,
> + unsigned char header_num)
> {
> unsigned char *p = io_u->buf;
> unsigned char c;
>
> - p += sizeof(*hdr);
> + p += header_num * hdr->len + sizeof(*hdr);
> c = crc7(p, hdr->len - sizeof(*hdr));
>
> if (c != hdr->crc7) {
> - log_err("crc7: verify failed at %llu/%lu\n", io_u->offset,
> io_u->buflen);
> + log_err("crc7: verify failed at %llu/%u\n",
> + io_u->offset + header_num * hdr->len,
> + hdr->len);
> log_err("crc7: wanted %x, got %x\n", hdr->crc7, c);
> return 1;
> }
> @@ -59,16 +62,19 @@ static int verify_io_u_crc7(struct verif
> return 0;
> }
>
> -static int verify_io_u_crc16(struct verify_header *hdr, struct io_u
> *io_u)
> +static int verify_io_u_crc16(struct verify_header *hdr, struct io_u
> *io_u,
> + unsigned int header_num)
> {
> unsigned char *p = io_u->buf;
> unsigned short c;
>
> - p += sizeof(*hdr);
> + p += header_num * hdr->len + sizeof(*hdr);
> c = crc16(p, hdr->len - sizeof(*hdr));
>
> if (c != hdr->crc16) {
> - log_err("crc16: verify failed at %llu/%lu\n",
> io_u->offset, io_u->buflen);
> + log_err("crc16: verify failed at %llu/%u\n",
> + io_u->offset + header_num * hdr->len,
> + hdr->len);
> log_err("crc16: wanted %x, got %x\n", hdr->crc16, c);
> return 1;
> }
> @@ -76,15 +82,19 @@ static int verify_io_u_crc16(struct veri
> return 0;
> }
>
> -static int verify_io_u_crc64(struct verify_header *hdr, struct io_u
> *io_u)
> +static int verify_io_u_crc64(struct verify_header *hdr, struct io_u
> *io_u,
> + unsigned int header_num)
> {
> - unsigned char *p = io_u->buf + sizeof(*hdr);
> + unsigned char *p = io_u->buf;
> unsigned long long c;
>
> + p += header_num * hdr->len + sizeof(*hdr);
> c = crc64(p, hdr->len - sizeof(*hdr));
>
> if (c != hdr->crc64) {
> - log_err("crc64: verify failed at %llu/%lu\n",
> io_u->offset, io_u->buflen);
> + log_err("crc64: verify failed at %llu/%u\n",
> + io_u->offset + header_num * hdr->len,
> + hdr->len);
> log_err("crc64: wanted %llx, got %llx\n", hdr->crc64, c);
> return 1;
> }
> @@ -92,16 +102,19 @@ static int verify_io_u_crc64(struct veri
> return 0;
> }
>
> -static int verify_io_u_crc32(struct verify_header *hdr, struct io_u
> *io_u)
> +static int verify_io_u_crc32(struct verify_header *hdr, struct io_u
> *io_u,
> + unsigned int header_num)
> {
> unsigned char *p = io_u->buf;
> unsigned long c;
>
> - p += sizeof(*hdr);
> + p += header_num * hdr->len + sizeof(*hdr);
> c = crc32(p, hdr->len - sizeof(*hdr));
>
> if (c != hdr->crc32) {
> - log_err("crc32: verify failed at %llu/%lu\n",
> io_u->offset, io_u->buflen);
> + log_err("crc32: verify failed at %llu/%u\n",
> + io_u->offset + header_num * hdr->len,
> + hdr->len);
> log_err("crc32: wanted %lx, got %lx\n", hdr->crc32, c);
> return 1;
> }
> @@ -109,18 +122,22 @@ static int verify_io_u_crc32(struct veri
> return 0;
> }
>
> -static int verify_io_u_md5(struct verify_header *hdr, struct io_u *io_u)
> +static int verify_io_u_md5(struct verify_header *hdr, struct io_u *io_u,
> + unsigned int header_num)
> {
> - unsigned char *p = io_u->buf + sizeof(*hdr);
> + unsigned char *p = io_u->buf;
> uint32_t hash[MD5_HASH_WORDS];
> struct md5_ctx md5_ctx = {
> .hash = hash,
> };
>
> + p += header_num * hdr->len + sizeof(*hdr);
> md5_update(&md5_ctx, p, hdr->len - sizeof(*hdr));
>
> - if (memcmp(hdr->md5_digest, md5_ctx.hash, sizeof(hash))) {
> - log_err("md5: verify failed at %llu/%lu\n", io_u->offset,
> io_u->buflen);
> + if (memcmp(hdr->md5_digest, md5_ctx.hash, sizeof(md5_ctx.hash))) {
> + log_err("md5: verify failed at %llu/%u\n",
> + io_u->offset + header_num * hdr->len,
> + hdr->len);
> hexdump(hdr->md5_digest, sizeof(hdr->md5_digest));
> hexdump(md5_ctx.hash, sizeof(hash));
> return 1;
> @@ -131,40 +148,48 @@ static int verify_io_u_md5(struct verify
>
> int verify_io_u(struct thread_data *td, struct io_u *io_u)
> {
> - struct verify_header *hdr = (struct verify_header *) io_u->buf;
> + unsigned char *p = (unsigned char*) io_u->buf;
> + struct verify_header *hdr;
> + unsigned int hdr_inc, hdr_num = 0;
> int ret;
>
> if (td->o.verify == VERIFY_NULL || io_u->ddir != DDIR_READ)
> return 0;
>
> - if (hdr->fio_magic != FIO_HDR_MAGIC) {
> - log_err("Bad verify header %x\n", hdr->fio_magic);
> - return EIO;
> - }
> + hdr_inc = io_u->buflen;
> + if (td->o.header_interval)
> + hdr_inc = td->o.header_interval;
>
> - switch (hdr->verify_type) {
> - case VERIFY_MD5:
> - ret = verify_io_u_md5(hdr, io_u);
> - break;
> - case VERIFY_CRC64:
> - ret = verify_io_u_crc64(hdr, io_u);
> - break;
> - case VERIFY_CRC32:
> - ret = verify_io_u_crc32(hdr, io_u);
> - break;
> - case VERIFY_CRC16:
> - ret = verify_io_u_crc16(hdr, io_u);
> - break;
> - case VERIFY_CRC7:
> - ret = verify_io_u_crc7(hdr, io_u);
> - break;
> - default:
> - log_err("Bad verify type %u\n", hdr->verify_type);
> - ret = 1;
> - }
> + for (; p < (unsigned char*) io_u->buf + io_u->buflen; p +=
> hdr_inc) {
> + hdr = (struct verify_header*) p;
>
> - if (ret)
> - return EIO;
> + if (hdr->fio_magic != FIO_HDR_MAGIC) {
> + log_err("Bad verify header %x\n", hdr->fio_magic);
> + return EIO;
> + }
> +
> + switch (hdr->verify_type) {
> + case VERIFY_MD5:
> + ret = verify_io_u_md5(hdr, io_u, hdr_num);
> + break;
> + case VERIFY_CRC64:
> + ret = verify_io_u_crc64(hdr, io_u, hdr_num);
> + break;
> + case VERIFY_CRC32:
> + ret = verify_io_u_crc32(hdr, io_u, hdr_num);
> + break;
> + case VERIFY_CRC16:
> + ret = verify_io_u_crc16(hdr, io_u, hdr_num);
> + break;
> + case VERIFY_CRC7:
> + ret = verify_io_u_crc7(hdr, io_u, hdr_num);
> + break;
> + default:
> + log_err("Bad verify type %u\n", hdr->verify_type);
> + ret = 1;
> + }
> + hdr_num++;
> + }
>
> return 0;
> }
> @@ -206,38 +231,46 @@ void populate_verify_io_u(struct thread_
> {
> const unsigned int len = io_u->buflen - sizeof(struct
> verify_header);
> struct verify_header *hdr;
> - unsigned char *p;
> + unsigned char *p = io_u->buf, *data;
> + unsigned int data_len;
>
> if (td->o.verify == VERIFY_NULL)
> return;
>
> - hdr = (struct verify_header *) io_u->buf;
> - hdr->fio_magic = FIO_HDR_MAGIC;
> - hdr->len = io_u->buflen;
> - hdr->verify_type = td->o.verify;
> -
> - p = io_u->buf + sizeof(*hdr);
> fill_random_bytes(td, p, len);
>
> - switch (td->o.verify) {
> - case VERIFY_MD5:
> - fill_md5(hdr, p, len);
> - break;
> - case VERIFY_CRC64:
> - fill_crc64(hdr, p, len);
> - break;
> - case VERIFY_CRC32:
> - fill_crc32(hdr, p, len);
> - break;
> - case VERIFY_CRC16:
> - fill_crc16(hdr, p, len);
> - break;
> - case VERIFY_CRC7:
> - fill_crc7(hdr, p, len);
> - break;
> - default:
> - log_err("fio: bad verify type: %d\n", td->o.verify);
> - assert(0);
> + for (;p < (unsigned char*) io_u->buf + io_u->buflen; p +=
> hdr->len) {
> + hdr = (struct verify_header*) p;
> +
> + hdr->fio_magic = FIO_HDR_MAGIC;
> + hdr->verify_type = td->o.verify;
> + hdr->len = io_u->buflen;
> + if (td->o.header_interval)
> + hdr->len = td->o.header_interval;
> +
> + data = p + sizeof(*hdr);
> + data_len = hdr->len - sizeof(*hdr);
> + switch (td->o.verify) {
> + case VERIFY_MD5:
> + fill_md5(hdr, data, data_len);
> + break;
> + case VERIFY_CRC64:
> + fill_crc64(hdr, data, data_len);
> + break;
> + case VERIFY_CRC32:
> + fill_crc32(hdr, data, data_len);
> + break;
> + case VERIFY_CRC16:
> + fill_crc16(hdr, data, data_len);
> + break;
> + case VERIFY_CRC7:
> + fill_crc7(hdr, data, data_len);
> + break;
> + default:
> + log_err("fio: bad verify type: %d\n", td->o.verify
> );
> + assert(0);
> + }
> + memcpy(p, hdr, sizeof(hdr));
> }
> }
>
>
>
Received on Fri Jul 27 2007 - 22:28:21 CEST

This archive was generated by hypermail 2.2.0 : Fri Jul 27 2007 - 22:30:02 CEST