From 94289a2aa1839f523aa701053122ba6d3b8bac9b Mon Sep 17 00:00:00 2001 From: Matt Macy Date: Sun, 8 Dec 2019 04:19:05 +0000 Subject: [PATCH] MFC r351770,r352920-r352923 MFCs to dd appear to have been haphazard so selectively MFCing individually didn't work easily. Add conv=fsync flag to dd The fsync flag performs an fsync(2) on the output file before closing it. This will be useful for the ZFS test suite. Add conv=fdatasync flag to dd The fdatasync flag performs an fdatasync(2) on the output file before closing it. This will be useful for the ZFS test suite. dd: Check result of close(2) for errors close(2) can return errors from previous operations which should not be ignored. Add oflag=fsync and oflag=sync capability to dd Sets the O_FSYNC flag on the output file. oflag=fsync and oflag=sync are synonyms just as O_FSYNC and O_SYNC are synonyms. This functionality is intended to improve portability of dd commands in the ZFS test suite. Add iflag=fullblock to dd Normally, count=n means read(2) will be called n times on the input to dd. If the read() returns short, as may happen when reading from a pipe, fewer bytes will be copied from the input. With conv=sync the buffer is padded with zeros to fill the rest of the block. iflag=fullblock causes dd to continue reading until the block is full, so that count=n means n full blocks are copied. This flag is compatible with illumos and GNU dd and is used in the ZFS test suite. Submitted by: Ryan Moeller, Thomas Hurst Reviewed by: manpages, mmacy@ Sponsored by: iXsystems, Inc. --- bin/dd/args.c | 83 +++++++++++++++++++++++++++++++++++++++++++++---- bin/dd/dd.1 | 42 +++++++++++++++++++++++-- bin/dd/dd.c | 78 ++++++++++++++++++++++++++++++---------------- bin/dd/dd.h | 66 +++++++++++++++++++++------------------ bin/dd/extern.h | 2 +- 5 files changed, 205 insertions(+), 66 deletions(-) diff --git a/bin/dd/args.c b/bin/dd/args.c index f58551dec6a..3ca49d243ce 100644 --- a/bin/dd/args.c +++ b/bin/dd/args.c @@ -41,7 +41,7 @@ static char sccsid[] = "@(#)args.c 8.3 (Berkeley) 4/2/94"; #include __FBSDID("$FreeBSD$"); -#include +#include #include #include @@ -57,6 +57,8 @@ __FBSDID("$FreeBSD$"); static int c_arg(const void *, const void *); static int c_conv(const void *, const void *); +static int c_iflag(const void *, const void *); +static int c_oflag(const void *, const void *); static void f_bs(char *); static void f_cbs(char *); static void f_conv(char *); @@ -65,8 +67,10 @@ static void f_files(char *); static void f_fillchar(char *); static void f_ibs(char *); static void f_if(char *); +static void f_iflag(char *); static void f_obs(char *); static void f_of(char *); +static void f_oflag(char *); static void f_seek(char *); static void f_skip(char *); static void f_speed(char *); @@ -77,7 +81,7 @@ static off_t get_off_t(const char *); static const struct arg { const char *name; void (*f)(char *); - u_int set, noset; + uint64_t set, noset; } args[] = { { "bs", f_bs, C_BS, C_BS|C_IBS|C_OBS|C_OSYNC }, { "cbs", f_cbs, C_CBS, C_CBS }, @@ -87,9 +91,11 @@ static const struct arg { { "fillchar", f_fillchar, C_FILL, C_FILL }, { "ibs", f_ibs, C_IBS, C_BS|C_IBS }, { "if", f_if, C_IF, C_IF }, + { "iflag", f_iflag, 0, 0 }, { "iseek", f_skip, C_SKIP, C_SKIP }, { "obs", f_obs, C_OBS, C_BS|C_OBS }, { "of", f_of, C_OF, C_OF }, + { "oflag", f_oflag, 0, 0 }, { "oseek", f_seek, C_SEEK, C_SEEK }, { "seek", f_seek, C_SEEK, C_SEEK }, { "skip", f_skip, C_SKIP, C_SKIP }, @@ -256,6 +262,38 @@ f_if(char *arg) in.name = arg; } +static const struct iflag { + const char *name; + uint64_t set, noset; +} ilist[] = { + { "fullblock", C_IFULLBLOCK, C_SYNC }, +}; + +static void +f_iflag(char *arg) +{ + struct iflag *ip, tmp; + + while (arg != NULL) { + tmp.name = strsep(&arg, ","); + ip = bsearch(&tmp, ilist, nitems(ilist), sizeof(struct iflag), + c_iflag); + if (ip == NULL) + errx(1, "unknown iflag %s", tmp.name); + if (ddflags & ip->noset) + errx(1, "%s: illegal conversion combination", tmp.name); + ddflags |= ip->set; + } +} + +static int +c_iflag(const void *a, const void *b) +{ + + return (strcmp(((const struct iflag *)a)->name, + ((const struct iflag *)b)->name)); +} + static void f_obs(char *arg) { @@ -314,12 +352,14 @@ f_status(char *arg) static const struct conv { const char *name; - u_int set, noset; + uint64_t set, noset; const u_char *ctab; } clist[] = { { "ascii", C_ASCII, C_EBCDIC, e2a_POSIX }, { "block", C_BLOCK, C_UNBLOCK, NULL }, { "ebcdic", C_EBCDIC, C_ASCII, a2e_POSIX }, + { "fdatasync", C_FDATASYNC, 0, NULL }, + { "fsync", C_FSYNC, 0, NULL }, { "ibm", C_EBCDIC, C_ASCII, a2ibm_POSIX }, { "lcase", C_LCASE, C_UCASE, NULL }, { "noerror", C_NOERROR, 0, NULL }, @@ -334,7 +374,7 @@ static const struct conv { { "parset", C_PARSET, C_PARODD|C_PAREVEN|C_PARNONE, NULL}, { "sparse", C_SPARSE, 0, NULL }, { "swab", C_SWAB, 0, NULL }, - { "sync", C_SYNC, 0, NULL }, + { "sync", C_SYNC, C_IFULLBLOCK, NULL }, { "ucase", C_UCASE, C_LCASE, NULL }, { "unblock", C_UNBLOCK, C_BLOCK, NULL }, }; @@ -346,8 +386,8 @@ f_conv(char *arg) while (arg != NULL) { tmp.name = strsep(&arg, ","); - cp = bsearch(&tmp, clist, sizeof(clist) / sizeof(struct conv), - sizeof(struct conv), c_conv); + cp = bsearch(&tmp, clist, nitems(clist), sizeof(struct conv), + c_conv); if (cp == NULL) errx(1, "unknown conversion %s", tmp.name); if (ddflags & cp->noset) @@ -366,6 +406,37 @@ c_conv(const void *a, const void *b) ((const struct conv *)b)->name)); } +static const struct oflag { + const char *name; + uint64_t set; +} olist[] = { + { "fsync", C_OFSYNC }, + { "sync", C_OFSYNC }, +}; + +static void +f_oflag(char *arg) +{ + struct oflag *op, tmp; + + while (arg != NULL) { + tmp.name = strsep(&arg, ","); + op = bsearch(&tmp, olist, nitems(olist), sizeof(struct oflag), + c_oflag); + if (op == NULL) + errx(1, "unknown open flag %s", tmp.name); + ddflags |= op->set; + } +} + +static int +c_oflag(const void *a, const void *b) +{ + + return (strcmp(((const struct oflag *)a)->name, + ((const struct oflag *)b)->name)); +} + static intmax_t postfix_to_mult(const char expr) { diff --git a/bin/dd/dd.1 b/bin/dd/dd.1 index 64a0977b1c1..756a585d3b2 100644 --- a/bin/dd/dd.1 +++ b/bin/dd/dd.1 @@ -32,7 +32,7 @@ .\" @(#)dd.1 8.2 (Berkeley) 1/13/94 .\" $FreeBSD$ .\" -.Dd August 8, 2018 +.Dd March 26, 2019 .Dt DD 1 .Os .Sh NAME @@ -102,6 +102,22 @@ bytes instead of the default 512. Read input from .Ar file instead of the standard input. +.It Cm iflag Ns = Ns Ar value Ns Op , Ns Ar value ... +Where +.Cm value +is one of the symbols from the following list. +.Bl -tag -width "fullblock" +.It Cm fullblock +Reading from the input file may not obtain a full block. +When a read returns short, continue reading to fill the block. +Without this flag, +.Cm count +limits the number of times +.Xr read 2 +is called on the input rather than the number of blocks copied in full. +May not be combined with +.Cm conv=sync . +.El .It Cm iseek Ns = Ns Ar n Seek on the input file .Ar n @@ -123,6 +139,19 @@ If an initial portion of the output file is seeked past (see the .Cm oseek operand), the output file is truncated at that point. +.It Cm oflag Ns = Ns Ar value Ns Op , Ns Ar value ... +Where +.Cm value +is one of the symbols from the following list. +.Bl -tag -width "fsync" +.It Cm fsync +Set the O_FSYNC flag on the output file to make writes synchronous. +.It Cm sync +Set the O_SYNC flag on the output file to make writes synchronous. +This is synonymous with the +.Cm fsync +value. +.El .It Cm oseek Ns = Ns Ar n Seek on the output file .Ar n @@ -252,6 +281,14 @@ are maps used in historic and .No pre- Ns Bx 4.3 reno systems. +.It Cm fdatasync +Perform an +.Xr fdatasync 2 +on the output file before closing it. +.It Cm fsync +Perform an +.Xr fsync 2 +on the output file before closing it. .It Cm lcase Transform uppercase characters into lowercase characters. .It Cm pareven , parnone , parodd , parset @@ -427,7 +464,8 @@ if necessary, to a 1MiB boundary: .Xr mt 1 , .Xr recoverdisk 1 , .Xr tr 1 , -.Xr geom 4 +.Xr geom 4 , +.Xr trim 8 .Sh STANDARDS The .Nm diff --git a/bin/dd/dd.c b/bin/dd/dd.c index 46175fa4c8f..fd228f332b9 100644 --- a/bin/dd/dd.c +++ b/bin/dd/dd.c @@ -83,7 +83,7 @@ STAT st; /* statistics */ void (*cfunc)(void); /* conversion function */ uintmax_t cpy_cnt; /* # of blocks to copy */ static off_t pending = 0; /* pending seek if sparse */ -u_int ddflags = 0; /* conversion options */ +uint64_t ddflags = 0; /* conversion options */ size_t cbsz; /* conversion block size */ uintmax_t files_cnt = 1; /* # of files to copy */ const u_char *ctab; /* conversion table */ @@ -124,7 +124,8 @@ main(int argc __unused, char *argv[]) * descriptor explicitly so that the summary handler (called * from an atexit() hook) includes this work. */ - close(out.fd); + if (close(out.fd) == -1 && errno != EINTR) + err(1, "close"); exit(0); } @@ -142,6 +143,7 @@ static void setup(void) { u_int cnt; + int oflags; cap_rights_t rights; unsigned long cmds[] = { FIODTYPE, MTIOCTOP }; @@ -164,21 +166,34 @@ setup(void) errx(1, "files is not supported for non-tape devices"); cap_rights_set(&rights, CAP_FTRUNCATE, CAP_IOCTL, CAP_WRITE); + if (ddflags & (C_FDATASYNC | C_FSYNC)) + cap_rights_set(&rights, CAP_FSYNC); if (out.name == NULL) { /* No way to check for read access here. */ out.fd = STDOUT_FILENO; out.name = "stdout"; + if (ddflags & C_OFSYNC) { + oflags = fcntl(out.fd, F_GETFL); + if (oflags == -1) + err(1, "unable to get fd flags for stdout"); + oflags |= O_FSYNC; + if (fcntl(out.fd, F_SETFL, oflags) == -1) + err(1, "unable to set fd flags for stdout"); + } } else { -#define OFLAGS \ - (O_CREAT | (ddflags & (C_SEEK | C_NOTRUNC) ? 0 : O_TRUNC)) - out.fd = open(out.name, O_RDWR | OFLAGS, DEFFILEMODE); + oflags = O_CREAT; + if (!(ddflags & (C_SEEK | C_NOTRUNC))) + oflags |= O_TRUNC; + if (ddflags & C_OFSYNC) + oflags |= O_FSYNC; + out.fd = open(out.name, O_RDWR | oflags, DEFFILEMODE); /* * May not have read access, so try again with write only. * Without read we may have a problem if output also does * not support seeks. */ if (out.fd == -1) { - out.fd = open(out.name, O_WRONLY | OFLAGS, DEFFILEMODE); + out.fd = open(out.name, O_WRONLY | oflags, DEFFILEMODE); out.flags |= NOREAD; cap_rights_clear(&rights, CAP_READ); } @@ -393,13 +408,15 @@ dd_in(void) memset(in.dbp, 0, in.dbsz); } - n = read(in.fd, in.dbp, in.dbsz); - if (n == 0) { - in.dbrcnt = 0; + in.dbrcnt = 0; +fill: + n = read(in.fd, in.dbp + in.dbrcnt, in.dbsz - in.dbrcnt); + + /* EOF */ + if (n == 0 && in.dbrcnt == 0) return; - } - /* Read error. */ + /* Read error */ if (n == -1) { /* * If noerror not specified, die. POSIX requires that @@ -423,25 +440,25 @@ dd_in(void) /* If sync not specified, omit block and continue. */ if (!(ddflags & C_SYNC)) continue; + } - /* Read errors count as full blocks. */ - in.dbcnt += in.dbrcnt = in.dbsz; - ++st.in_full; + /* If conv=sync, use the entire block. */ + if (ddflags & C_SYNC) + n = in.dbsz; - /* Handle full input blocks. */ - } else if ((size_t)n == (size_t)in.dbsz) { - in.dbcnt += in.dbrcnt = n; - ++st.in_full; + /* Count the bytes read for this block. */ + in.dbrcnt += n; - /* Handle partial input blocks. */ - } else { - /* If sync, use the entire block. */ - if (ddflags & C_SYNC) - in.dbcnt += in.dbrcnt = in.dbsz; - else - in.dbcnt += in.dbrcnt = n; + /* Count the number of full and partial blocks. */ + if (in.dbrcnt == in.dbsz) + ++st.in_full; + else if (ddflags & C_IFULLBLOCK && n != 0) + goto fill; /* these don't count */ + else ++st.in_part; - } + + /* Count the total bytes read for this file. */ + in.dbcnt += in.dbrcnt; /* * POSIX states that if bs is set and no other conversions @@ -463,6 +480,7 @@ dd_in(void) swapbytes(in.dbp, (size_t)n); } + /* Advance to the next block. */ in.dbp += in.dbrcnt; (*cfunc)(); if (need_summary) @@ -505,6 +523,14 @@ dd_close(void) if (ftruncate(out.fd, out.seek_offset) == -1) err(1, "truncating %s", out.name); } + + if (ddflags & C_FSYNC) { + if (fsync(out.fd) == -1) + err(1, "fsyncing %s", out.name); + } else if (ddflags & C_FDATASYNC) { + if (fdatasync(out.fd) == -1) + err(1, "fdatasyncing %s", out.name); + } } void diff --git a/bin/dd/dd.h b/bin/dd/dd.h index 8090252923f..a39f879830d 100644 --- a/bin/dd/dd.h +++ b/bin/dd/dd.h @@ -70,37 +70,41 @@ typedef struct { } STAT; /* Flags (in ddflags). */ -#define C_ASCII 0x00000001 -#define C_BLOCK 0x00000002 -#define C_BS 0x00000004 -#define C_CBS 0x00000008 -#define C_COUNT 0x00000010 -#define C_EBCDIC 0x00000020 -#define C_FILES 0x00000040 -#define C_IBS 0x00000080 -#define C_IF 0x00000100 -#define C_LCASE 0x00000200 -#define C_NOERROR 0x00000400 -#define C_NOTRUNC 0x00000800 -#define C_OBS 0x00001000 -#define C_OF 0x00002000 -#define C_OSYNC 0x00004000 -#define C_PAREVEN 0x00008000 -#define C_PARNONE 0x00010000 -#define C_PARODD 0x00020000 -#define C_PARSET 0x00040000 -#define C_SEEK 0x00080000 -#define C_SKIP 0x00100000 -#define C_SPARSE 0x00200000 -#define C_SWAB 0x00400000 -#define C_SYNC 0x00800000 -#define C_UCASE 0x01000000 -#define C_UNBLOCK 0x02000000 -#define C_FILL 0x04000000 -#define C_STATUS 0x08000000 -#define C_NOXFER 0x10000000 -#define C_NOINFO 0x20000000 -#define C_PROGRESS 0x40000000 +#define C_ASCII 0x0000000000000001ULL +#define C_BLOCK 0x0000000000000002ULL +#define C_BS 0x0000000000000004ULL +#define C_CBS 0x0000000000000008ULL +#define C_COUNT 0x0000000000000010ULL +#define C_EBCDIC 0x0000000000000020ULL +#define C_FILES 0x0000000000000040ULL +#define C_IBS 0x0000000000000080ULL +#define C_IF 0x0000000000000100ULL +#define C_LCASE 0x0000000000000200ULL +#define C_NOERROR 0x0000000000000400ULL +#define C_NOTRUNC 0x0000000000000800ULL +#define C_OBS 0x0000000000001000ULL +#define C_OF 0x0000000000002000ULL +#define C_OSYNC 0x0000000000004000ULL +#define C_PAREVEN 0x0000000000008000ULL +#define C_PARNONE 0x0000000000010000ULL +#define C_PARODD 0x0000000000020000ULL +#define C_PARSET 0x0000000000040000ULL +#define C_SEEK 0x0000000000080000ULL +#define C_SKIP 0x0000000000100000ULL +#define C_SPARSE 0x0000000000200000ULL +#define C_SWAB 0x0000000000400000ULL +#define C_SYNC 0x0000000000800000ULL +#define C_UCASE 0x0000000001000000ULL +#define C_UNBLOCK 0x0000000002000000ULL +#define C_FILL 0x0000000004000000ULL +#define C_STATUS 0x0000000008000000ULL +#define C_NOXFER 0x0000000010000000ULL +#define C_NOINFO 0x0000000020000000ULL +#define C_PROGRESS 0x0000000040000000ULL +#define C_FSYNC 0x0000000080000000ULL +#define C_FDATASYNC 0x0000000100000000ULL +#define C_OFSYNC 0x0000000200000000ULL +#define C_IFULLBLOCK 0x0000000400000000ULL #define C_PARITY (C_PAREVEN | C_PARODD | C_PARNONE | C_PARSET) diff --git a/bin/dd/extern.h b/bin/dd/extern.h index 45e3fb1b280..07c08e2ef8f 100644 --- a/bin/dd/extern.h +++ b/bin/dd/extern.h @@ -58,7 +58,7 @@ extern STAT st; extern void (*cfunc)(void); extern uintmax_t cpy_cnt; extern size_t cbsz; -extern u_int ddflags; +extern uint64_t ddflags; extern size_t speed; extern uintmax_t files_cnt; extern const u_char *ctab; -- 2.45.0